def retreive_key(self, *data):
time.sleep(random.randint(0, 1000) / 1000)
dict = data[0]
from_node = data[1]
val = self.kv.get(dict.key, None)
response = Request("ACK-GET", dict.key, val, dict.request)
Messaging.send_message(self, from_node, response)
def perform_put(self, *data):
dict = data[0]
key = dict.key
preference_list = self.hash_ring.get_node(key, self.failed_nodes)
if (self not in preference_list):
coordinator = preference_list[0].id
dict = Request("FORWARD-PUT", dict.key, dict.value,
generate_random_number(), dict.client)
Messaging.send_message(self, coordinator, dict)
time.sleep(3)
if REQUESTS.get(dict.request, False) != True:
#print("Timedout PUT")
dict.action = "PUT"
dict.request = generate_random_number()
self.socket.settimeout(None)
self.failed_nodes.append(coordinator)
self.perform_put(dict)
else:
self.vector_clock.update(self.id, self.get_sequence_no())
metadata = deepcopy(self.vector_clock)
if dict.value[1] > metadata:
metadata = dict.value[1]
dict.value = (dict.value[0], metadata)
dict.request = generate_random_number()
Messaging.broadcast_put(self, preference_list, dict)
def handoff(self):
for node in self.check_for_sync:
keys = list(self.sync_kv[node].keys())
values = list(self.sync_kv[node].values())
synchronize = Request("SYNC", keys, values,
generate_random_number())
Messaging.send_message(self, node, synchronize)
def exposed_CLIBroadcast(self, sUsername, sMessage):
"""Gets the state for the CLI"""
global gCoreServerLock
global gDatabase
global gMessaging
bLocked = False
try:
# Acquire the lock
gCoreServerLock.Acquire(lockTimeout)
bLocked = True
log.info("CoreServerService.CLIBroadcast(" + sMessage + ")")
# Create the messaging object and broadcast the message
if gMessaging == None:
gMessaging = Messaging("System", gDatabase)
gMessaging.broadcastMessage(sMessage)
bResult = True
except Exception as ex:
# Log the error
log.error("CoreServerService.CLIBroadcast() failed: " + str(ex))
log.error("Traceback:\r\n%s\r\n" % traceback.format_exc())
gMessaging = None
bResult = False
# Release the lock and return
if bLocked:
gCoreServerLock.Release()
return bResult
def exposed_CLIBroadcast(self, sUsername, sMessage):
"""Gets the state for the CLI"""
global gCoreServerLock
global gDatabase
global gMessaging
bLocked = False
try:
# Acquire the lock
gCoreServerLock.Acquire(lockTimeout)
bLocked = True
log.info("CoreServerService.CLIBroadcast(" + sMessage + ")")
# Create the messaging object and broadcast the message
if gMessaging == None:
gMessaging = Messaging("System", gDatabase)
gMessaging.broadcastMessage(sMessage)
bResult = True
except Exception as ex:
# Log the error
log.error("CoreServerService.CLIBroadcast() failed: " + str(ex))
log.error("Traceback:\r\n%s\r\n" % traceback.format_exc())
gMessaging = None
bResult = False
# Release the lock and return
if bLocked:
gCoreServerLock.Release()
return bResult
def resurrect(self, node):
dragonsblood = Request("REVIVE", None)
Messaging.send_message(self, node, dragonsblood)
# client = Client(1)
# client.put_req('x',1)
# time.sleep(1)
# client.get_req('x')
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel):
"""Constructor"""
# Call base class init
Thread.__init__(self)
# Initialize variables
self.username = sRemoteUser
self.sourceSequenceID = nSourceSequenceID
self.sourceComponentID = 0
self.runSequenceID = 0
self.runComponentID = 0
self.systemModel = pSystemModel
self.initializing = True
self.running = False
self.startComponentID = 0
self.userSourceIDs = True
self.runWillPause = False
self.runIsPaused = False
self.runAborted = False
self.showAbortPrompt = False
# Create database connection and sequence manager
self.database = DBComm()
self.database.Connect()
self.sequenceManager = SequenceManager(self.database)
# Create messaging object
self.messaging = Messaging(self.username, self.database)
# Fetch the sequence from the database and make sure it is valid
self.sourceSequence = self.sequenceManager.GetSequence(
self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Cannot run an invalid sequence (" +
self.sourceSequenceID + ")")
# Create a new sequence in the run history
log.debug("Starting run of sequence " + str(self.sourceSequenceID) +
" (" + self.sourceSequence["metadata"]["name"] + ")")
pConfiguration = self.database.GetConfiguration(self.username)
self.runSequenceID = self.database.CreateSequence(
self.username, self.sourceSequence["metadata"]["name"],
self.sourceSequence["metadata"]["comment"], "History",
pConfiguration["reactors"], pConfiguration["reagentsperreactor"])
# Copy the cassettes
for pComponent in self.sourceSequence["components"]:
if pComponent["componenttype"] == Cassette.componentType:
pUnitOperation = UnitOperations.createFromComponent(
self.sourceSequenceID, pComponent, self.username,
self.database)
pUnitOperation.copyComponent(self.sourceSequenceID,
self.runSequenceID)
def perform_store(self, data):
time.sleep(random.randint(0, 1000) / 1000)
dict = data[0]
addr = data[1]
self.vector_clock.update(self.id, self.get_sequence_no())
self.vector_clock = self.vector_clock.converge(
[self.vector_clock, dict.value[1]])
if dict.key not in self.kv:
self.kv[dict.key] = list()
self.kv[dict.key].append(dict.value)
self.kv[dict.key] = self.perform_syntactic_reconcilation(
self.kv[dict.key])
dict = Request("ACK-PUT", None, None, dict.request)
Messaging.send_message(self, PORT_TO_ID[addr[1]], dict)
def PrintAccessors(self, msgClass):
msg = msgClass()
for fieldInfo in msgClass.fields:
txt = "body.%s.%s: " % (msgClass.__name__, fieldInfo.name)
if (fieldInfo.count == 1):
txt += str(Messaging.get(msg, fieldInfo))
else:
for i in range(0, fieldInfo.count):
#print("body.",msgClass.__name__, ".", method.__name__, "[",i,"] = ", method(msg,i), " #", method.__doc__, "in", method.units)
txt += str(Messaging.get(msg, fieldInfo, i))
if (i < fieldInfo.count - 1):
txt += ", "
txt += " # " + fieldInfo.description + " in " + fieldInfo.units
print(txt)
def test_accessors(self):
msgclass = self.msgLib.MsgClassFromName["Network.Connect"]
sameMsgClass = self.msgLib.Messages.Network.Connect
self.assertEqual(msgclass, sameMsgClass)
expected = "Testing"
testMsg = msgclass()
Messaging.set(testMsg, msgclass.fields[0], expected)
observed = testMsg.GetName()
self.assertMultiLineEqual(expected, observed)
expected = "MoreTesting"
testMsg.SetName(expected)
observed = Messaging.get(testMsg, msgclass.fields[0])
self.assertMultiLineEqual(expected, observed)
def perform_get(self, dict):
key = dict.key
preference_list = self.hash_ring.get_node(key, self.failed_nodes)
if (self not in preference_list):
coordinator = preference_list[0].id
dict = Request("FORWARD-GET", dict.key, dict.value,
generate_random_number(), dict.client)
Messaging.send_message(self, coordinator, dict)
time.sleep(3)
if REQUESTS.get(dict.request, False) != True:
#print("Timedout GET")
dict.action = "GET"
dict.request = generate_random_number()
self.failed_nodes.append(coordinator)
self.perform_get(dict)
else:
dict.request = generate_random_number()
Messaging.broadcast_get(self, preference_list, dict)
def put(self, key, value, retries=0):
while retries < MAX_RETRIES:
dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)]
vc = VectorClock()
vc.update(dynamo_node, 0)
value1 = (value, self.datastore.get(key, vc))
#print("Client value is "+str(value1))
put_request = Request("PUT", key, value1, None, self.port)
self.socket.settimeout(5)
Messaging.send_message(self, dynamo_node, put_request)
try:
while True:
data, addr = self.socket.recvfrom(40960)
if data:
self.socket.settimeout(None)
return pickle.loads(data)
except Exception:
self.socket.settimeout(None)
retries += 1
def get(self, key, retries=0):
while retries < MAX_RETRIES:
get_request = Request("GET", key, None, None, self.port)
dynamo_node = self.nodes[random.randint(0, len(self.nodes) - 1)]
self.socket.settimeout(5)
Messaging.send_message(self, dynamo_node, get_request)
try:
while True:
data, addr = self.socket.recvfrom(40960)
if data:
data = pickle.loads(data)
if (data == "FAILURE"):
# print("FAILURE")
return -INF
result, self.datastore[
key] = self.perform_semantic_reconcilation(data)
#print(str(result)+" "+str(self.datastore[key].clock))
self.socket.settimeout(None)
return result
except Exception:
self.socket.settimeout(None)
retries += 1
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel):
"""Constructor"""
# Call base class init
Thread.__init__(self)
# Initialize variables
self.username = sRemoteUser
self.sourceSequenceID = nSourceSequenceID
self.sourceComponentID = 0
self.runSequenceID = 0
self.runComponentID = 0
self.systemModel = pSystemModel
self.initializing = True
self.running = False
self.startComponentID = 0
self.userSourceIDs = True
self.runWillPause = False
self.runIsPaused = False
self.runAborted = False
self.showAbortPrompt = False
# Create database connection and sequence manager
self.database = DBComm()
self.database.Connect()
self.sequenceManager = SequenceManager(self.database)
# Create messaging object
self.messaging = Messaging(self.username, self.database)
# Fetch the sequence from the database and make sure it is valid
self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")")
# Create a new sequence in the run history
log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")")
pConfiguration = self.database.GetConfiguration(self.username)
self.runSequenceID = self.database.CreateSequence(self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History",
pConfiguration["reactors"], pConfiguration["reagentsperreactor"])
# Copy the cassettes
for pComponent in self.sourceSequence["components"]:
if pComponent["componenttype"] == Cassette.componentType:
pUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pComponent, self.username, self.database)
pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
def run(self):
if self.broadcastFlag:
pMessaging = Messaging(self.username, self.database)
pMessaging.broadcastMessage(urllib.unquote(self.userMessage))
def kill(self, node=None):
if node is None:
node = self.nodes[random.randint(0, len(self.nodes) - 1)]
kill_switch = Request("EXIT", None)
Messaging.send_message(self, node, kill_switch)
return node
class Sequence(Thread):
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel):
"""Constructor"""
# Call base class init
Thread.__init__(self)
# Initialize variables
self.username = sRemoteUser
self.sourceSequenceID = nSourceSequenceID
self.sourceComponentID = 0
self.runSequenceID = 0
self.runComponentID = 0
self.systemModel = pSystemModel
self.initializing = True
self.running = False
self.startComponentID = 0
self.userSourceIDs = True
self.runWillPause = False
self.runIsPaused = False
self.runAborted = False
self.showAbortPrompt = False
# Create database connection and sequence manager
self.database = DBComm()
self.database.Connect()
self.sequenceManager = SequenceManager(self.database)
# Create messaging object
self.messaging = Messaging(self.username, self.database)
# Fetch the sequence from the database and make sure it is valid
self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Cannot run an invalid sequence (" + self.sourceSequenceID + ")")
# Create a new sequence in the run history
log.debug("Starting run of sequence " + str(self.sourceSequenceID) + " (" + self.sourceSequence["metadata"]["name"] + ")")
pConfiguration = self.database.GetConfiguration(self.username)
self.runSequenceID = self.database.CreateSequence(self.username, self.sourceSequence["metadata"]["name"], self.sourceSequence["metadata"]["comment"], "History",
pConfiguration["reactors"], pConfiguration["reagentsperreactor"])
# Copy the cassettes
for pComponent in self.sourceSequence["components"]:
if pComponent["componenttype"] == Cassette.componentType:
pUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pComponent, self.username, self.database)
pUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
def setStartComponent(self, nComponentID):
"""Sets the first component of the run"""
self.startComponentID = nComponentID
def getIDs(self):
"""Return the appropriate sequence and component IDs"""
if self.userSourceIDs:
return (self.sourceSequenceID, self.sourceComponentID)
else:
return (self.runSequenceID, self.runComponentID)
def pauseRun(self):
"""Flags the running sequence to pause at the start of the next unit operation"""
if not self.running:
raise Exception("Sequence not running, cannot pause")
if self.runWillPause:
raise Exception("Sequence already will pause, cannot pause")
if self.runIsPaused:
raise Exception("Sequence is already paused, cannot pause")
self.runWillPause = True
def continueRun(self):
"""Continues a paused sequence"""
if not self.running:
raise Exception("Sequence not running, cannot continue")
if not self.runWillPause and not self.runIsPaused:
raise Exception("Sequence run not paused, cannot continue")
self.runWillPause = False
self.runIsPaused = False
def willRunPause(self):
"""Returns true if the sequence run is flagged to pause, false otherwise"""
return self.runWillPause
def isRunPaused(self):
"""Returns true if the sequence run is paused, false otherwise"""
return self.runIsPaused
def abortRun(self):
"""Aborts the current sequence run"""
self.runAborted = True
self.showAbortPrompt = False
def isRunComplete(self):
"""Returns true if the sequence run has completed and is waiting at the summary unit operation, false otherwise"""
return not self.userSourceIDs
def setShowAbortPrompt(self, bShowAbortPrompt):
"""Sets the flag that indicates if the abort sequence run prompt should be set in the run state"""
self.showAbortPrompt = bShowAbortPrompt
def getShowAbortPrompt(self):
"""Returns the flag that indicates if the abort sequence run prompt should be set in the run state"""
return self.showAbortPrompt
def run(self):
"""Thread entry point"""
sRunError = ""
self.userSourceIDs = True
log.debug("Running Sequence")
try:
# Main sequence run loop
log.debug("sourceSequence -> %s" % str(self.sourceSequence))
sMessage = "Run of sequence \"" + self.sourceSequence["metadata"]["name"] + "\" started."
log.debug(sMessage)
self.messaging.broadcastMessage(sMessage)
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
# Get the next component
pSourceComponent = self.sourceSequence["components"][nCurrentComponent]
# Skip components until we find our start component
self.sourceComponentID = pSourceComponent["id"]
if self.initializing and (self.startComponentID != 0) and (self.sourceComponentID != self.startComponentID):
log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Update our initializing and running flags
self.initializing = False
self.running = True
# Ignore any previous summary component
if pSourceComponent["componenttype"] == Summary.componentType:
log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Create and run the next unit operation
log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" +
pSourceComponent["componenttype"] + ")")
pSourceUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel)
self.runComponentID = pSourceUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
pRunComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID)
pRunUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel)
pRunUnitOperation.setDaemon(True)
pRunUnitOperation.start()
self.systemModel.SetUnitOperation(pRunUnitOperation)
log.debug( "Unit Op is alive")
# Wait until the operation completes or we receive an abort signal
while pRunUnitOperation.is_alive() and not self.runAborted:
log.debug( "Unit Op check alive")
time.sleep(0.25)
self.systemModel.SetUnitOperation(None)
if self.runAborted:
log.error( "Unit Op ABORTED!")
pRunUnitOperation.setAbort()
raise Exception("Run aborted")
# Check for unit operation error
log.debug( "Unit check for error")
sError = pRunUnitOperation.getError()
if sError != "":
log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__)
log.error( "Unit operation failed: " + sError)
raise Exception(sError)
log.debug("Prepare to update operation details")
# Update the unit operation details in the database
UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel)
log.debug("After updateToComponent")
self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent)
log.debug("After updateComponent")
log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")")
self.sourceComponentID = 0
self.runComponentID = 0
# Check if the user paused the sequence for editing
if self.runWillPause:
# Pause until editing is complete
log.debug("Pausing run for editing")
self.runWillPause = False
self.runIsPaused = True
while self.runIsPaused and not self.runAborted:
time.sleep(0.25)
if self.runAborted:
raise Exception("Sequence aborted")
log.debug("Continuing paused run")
# Reload the sequence and make sure it is still valid
self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Edited sequence is no longer valid (" + self.sourceSequenceID + ")")
# Scan until we find the unit operation we just executed
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
if self.sourceSequence["components"][nCurrentComponent]["id"] == pSourceComponent["id"]:
break
nCurrentComponent += 1
if nCurrentComponent == nComponentCount:
raise Exception("Failed to find previous component in edited sequence")
# Advance to the next component
nCurrentComponent += 1
except Exception as ex:
log.error("Sequence run failed: " + str(ex))
log.error("Trace Back %s" % traceback.format_exc())
sRunError = str(ex)
# Add the Summary unit operation to the sequence
pSummaryComponent = Summary.createNewComponent(sRunError)
self.runComponentID = self.database.CreateComponent(self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent))
# Fully validate the run sequence
self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID)
# Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence
self.userSourceIDs = False
# Instantiate and start the summary unit operation
log.info("Starting summary unit operation")
pSummaryComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID)
pSummaryUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel)
pSummaryUnitOperation.setDaemon(True)
pSummaryUnitOperation.start()
self.systemModel.SetUnitOperation(pSummaryUnitOperation)
# Broadcast the summary unit operation message
self.messaging.broadcastMessage(pSummaryComponent["message"])
# Wait until the operation completes
while pSummaryUnitOperation.is_alive():
time.sleep(0.25)
log.info("Summary unit operation complete")
self.runComponentID = 0
# Run complete
log.debug("Run stopped")
self.running = False
def setUpClass(cls):
print("----------- Running setup")
cls.msgLib = Messaging(None, 0, "NetworkHeader")
class Sequence(Thread):
def __init__(self, sRemoteUser, nSourceSequenceID, pSystemModel):
"""Constructor"""
# Call base class init
Thread.__init__(self)
# Initialize variables
self.username = sRemoteUser
self.sourceSequenceID = nSourceSequenceID
self.sourceComponentID = 0
self.runSequenceID = 0
self.runComponentID = 0
self.systemModel = pSystemModel
self.initializing = True
self.running = False
self.startComponentID = 0
self.userSourceIDs = True
self.runWillPause = False
self.runIsPaused = False
self.runAborted = False
self.showAbortPrompt = False
# Create database connection and sequence manager
self.database = DBComm()
self.database.Connect()
self.sequenceManager = SequenceManager(self.database)
# Create messaging object
self.messaging = Messaging(self.username, self.database)
# Fetch the sequence from the database and make sure it is valid
self.sourceSequence = self.sequenceManager.GetSequence(
self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Cannot run an invalid sequence (" +
self.sourceSequenceID + ")")
# Create a new sequence in the run history
log.debug("Starting run of sequence " + str(self.sourceSequenceID) +
" (" + self.sourceSequence["metadata"]["name"] + ")")
pConfiguration = self.database.GetConfiguration(self.username)
self.runSequenceID = self.database.CreateSequence(
self.username, self.sourceSequence["metadata"]["name"],
self.sourceSequence["metadata"]["comment"], "History",
pConfiguration["reactors"], pConfiguration["reagentsperreactor"])
# Copy the cassettes
for pComponent in self.sourceSequence["components"]:
if pComponent["componenttype"] == Cassette.componentType:
pUnitOperation = UnitOperations.createFromComponent(
self.sourceSequenceID, pComponent, self.username,
self.database)
pUnitOperation.copyComponent(self.sourceSequenceID,
self.runSequenceID)
def setStartComponent(self, nComponentID):
"""Sets the first component of the run"""
self.startComponentID = nComponentID
def getIDs(self):
"""Return the appropriate sequence and component IDs"""
if self.userSourceIDs:
return (self.sourceSequenceID, self.sourceComponentID)
else:
return (self.runSequenceID, self.runComponentID)
def pauseRun(self):
"""Flags the running sequence to pause at the start of the next unit operation"""
if not self.running:
raise Exception("Sequence not running, cannot pause")
if self.runWillPause:
raise Exception("Sequence already will pause, cannot pause")
if self.runIsPaused:
raise Exception("Sequence is already paused, cannot pause")
self.runWillPause = True
def continueRun(self):
"""Continues a paused sequence"""
if not self.running:
raise Exception("Sequence not running, cannot continue")
if not self.runWillPause and not self.runIsPaused:
raise Exception("Sequence run not paused, cannot continue")
self.runWillPause = False
self.runIsPaused = False
def willRunPause(self):
"""Returns true if the sequence run is flagged to pause, false otherwise"""
return self.runWillPause
def isRunPaused(self):
"""Returns true if the sequence run is paused, false otherwise"""
return self.runIsPaused
def abortRun(self):
"""Aborts the current sequence run"""
self.runAborted = True
self.showAbortPrompt = False
def isRunComplete(self):
"""Returns true if the sequence run has completed and is waiting at the summary unit operation, false otherwise"""
return not self.userSourceIDs
def setShowAbortPrompt(self, bShowAbortPrompt):
"""Sets the flag that indicates if the abort sequence run prompt should be set in the run state"""
self.showAbortPrompt = bShowAbortPrompt
def getShowAbortPrompt(self):
"""Returns the flag that indicates if the abort sequence run prompt should be set in the run state"""
return self.showAbortPrompt
def run(self):
"""Thread entry point"""
sRunError = ""
self.userSourceIDs = True
log.debug("Running Sequence")
try:
# Main sequence run loop
log.debug("sourceSequence -> %s" % str(self.sourceSequence))
sMessage = "Run of sequence \"" + self.sourceSequence["metadata"][
"name"] + "\" started."
log.debug(sMessage)
self.messaging.broadcastMessage(sMessage)
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
# Get the next component
pSourceComponent = self.sourceSequence["components"][
nCurrentComponent]
# Skip components until we find our start component
self.sourceComponentID = pSourceComponent["id"]
if self.initializing and (self.startComponentID != 0) and (
self.sourceComponentID != self.startComponentID):
log.debug("Skipping unit operation (" +
pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Update our initializing and running flags
self.initializing = False
self.running = True
# Ignore any previous summary component
if pSourceComponent["componenttype"] == Summary.componentType:
log.debug("Skipping unit operation (" +
pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Create and run the next unit operation
log.debug("Starting unit operation " +
str(self.sourceComponentID) + " (" +
pSourceComponent["componenttype"] + ")")
pSourceUnitOperation = UnitOperations.createFromComponent(
self.sourceSequenceID, pSourceComponent, self.username,
self.sequenceManager.database, self.systemModel)
self.runComponentID = pSourceUnitOperation.copyComponent(
self.sourceSequenceID, self.runSequenceID)
pRunComponent = self.sequenceManager.GetComponent(
self.username, self.runComponentID, self.runSequenceID)
pRunUnitOperation = UnitOperations.createFromComponent(
self.runSequenceID, pRunComponent, self.username,
self.sequenceManager.database, self.systemModel)
pRunUnitOperation.setDaemon(True)
pRunUnitOperation.start()
self.systemModel.SetUnitOperation(pRunUnitOperation)
log.debug("Unit Op is alive")
# Wait until the operation completes or we receive an abort signal
while pRunUnitOperation.is_alive() and not self.runAborted:
log.debug("Unit Op check alive")
time.sleep(0.25)
self.systemModel.SetUnitOperation(None)
if self.runAborted:
log.error("Unit Op ABORTED!")
pRunUnitOperation.setAbort()
raise Exception("Run aborted")
# Check for unit operation error
log.debug("Unit check for error")
sError = pRunUnitOperation.getError()
if sError != "":
log.error("UnitOperation: %s" %
pRunUnitOperation.__class__.__name__)
log.error("Unit operation failed: " + sError)
raise Exception(sError)
log.debug("Prepare to update operation details")
# Update the unit operation details in the database
UnitOperations.updateToComponent(pRunUnitOperation,
self.runSequenceID,
pRunComponent, self.username,
self.sequenceManager.database,
self.systemModel)
log.debug("After updateToComponent")
self.sequenceManager.UpdateComponent(self.username,
self.runSequenceID,
self.runComponentID, None,
pRunComponent)
log.debug("After updateComponent")
log.debug("Completed unit operation (" +
pRunComponent["componenttype"] + ")")
self.sourceComponentID = 0
self.runComponentID = 0
# Check if the user paused the sequence for editing
if self.runWillPause:
# Pause until editing is complete
log.debug("Pausing run for editing")
self.runWillPause = False
self.runIsPaused = True
while self.runIsPaused and not self.runAborted:
time.sleep(0.25)
if self.runAborted:
raise Exception("Sequence aborted")
log.debug("Continuing paused run")
# Reload the sequence and make sure it is still valid
self.sourceSequence = self.sequenceManager.GetSequence(
self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions(
"Edited sequence is no longer valid (" +
self.sourceSequenceID + ")")
# Scan until we find the unit operation we just executed
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
if self.sourceSequence["components"][
nCurrentComponent]["id"] == pSourceComponent[
"id"]:
break
nCurrentComponent += 1
if nCurrentComponent == nComponentCount:
raise Exception(
"Failed to find previous component in edited sequence"
)
# Advance to the next component
nCurrentComponent += 1
except Exception as ex:
log.error("Sequence run failed: " + str(ex))
log.error("Trace Back %s" % traceback.format_exc())
sRunError = str(ex)
# Add the Summary unit operation to the sequence
pSummaryComponent = Summary.createNewComponent(sRunError)
self.runComponentID = self.database.CreateComponent(
self.username, self.runSequenceID, pSummaryComponent["type"],
pSummaryComponent["note"], json.dumps(pSummaryComponent))
# Fully validate the run sequence
self.sequenceManager.ValidateSequenceFull(self.username,
self.runSequenceID)
# Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence
self.userSourceIDs = False
# Instantiate and start the summary unit operation
log.info("Starting summary unit operation")
pSummaryComponent = self.sequenceManager.GetComponent(
self.username, self.runComponentID, self.runSequenceID)
pSummaryUnitOperation = UnitOperations.createFromComponent(
self.runSequenceID, pSummaryComponent, self.username,
self.sequenceManager.database, self.systemModel)
pSummaryUnitOperation.setDaemon(True)
pSummaryUnitOperation.start()
self.systemModel.SetUnitOperation(pSummaryUnitOperation)
# Broadcast the summary unit operation message
self.messaging.broadcastMessage(pSummaryComponent["message"])
# Wait until the operation completes
while pSummaryUnitOperation.is_alive():
time.sleep(0.25)
log.info("Summary unit operation complete")
self.runComponentID = 0
# Run complete
log.debug("Run stopped")
self.running = False
def checkIfAlive(self):
for node in self.failed_nodes:
ping = Request("PING", None)
Messaging.send_message(self, node, ping)
def run(self):
try:
while True:
data, addr = self.socket.recvfrom(MAX_MESSAGE_SIZE)
dict = pickle.loads(data)
# #print(self.id + " received data from" + str(addr[1]))
if self.state == 1:
if dict.action == "PUT":
#print(self.id+" received PUT "+str(dict.key)+" "+str(dict.value)+" "+str(dict.client))
thread1 = Thread(target=self.perform_put,
args=(dict, ))
thread1.start()
if dict.action == "STORE":
#print(self.id + " received STORE"+str(dict.key)+" "+str(dict.value[0]))
thread2 = Thread(target=self.perform_store,
args=([dict, addr], ))
thread2.start()
#print(self.id+" "+self.string(self.kv))
if dict.action == "GET":
#print(self.id+" Received Get")
thread3 = Thread(target=self.perform_get,
args=(dict, ))
thread3.start()
if dict.action == "FETCH":
#print(self.id+" Received Fetch")
thread4 = Thread(target=self.retreive_key,
args=[dict, PORT_TO_ID[addr[1]]])
thread4.start()
if dict.action == "EXIT":
#print(self.id+" Will Fail now")
self.state = 0
if dict.action == "FORWARD-PUT":
#print(self.id+"Received Forward")
dict.action = "PUT"
response = Request("ACK-FORWARD-PUT", None, None,
dict.request)
self.socket.sendto(pickle.dumps(response), addr)
thread5 = Thread(target=self.perform_put,
args=(dict, ))
thread5.start()
#self.perform_put(dict)
if dict.action == "FORWARD-GET":
#print(self.id+"Received Forward")
dict.action = "GET"
response = Request("ACK-FORWARD-GET", None, None,
dict.request)
self.socket.sendto(pickle.dumps(response), addr)
threada = Thread(target=self.perform_get, args=[dict])
threada.start()
if dict.action == "PING":
#print(self.id+"Received Ping")
response = Request("PONG", None)
Messaging.send_message(self, PORT_TO_ID[addr[1]],
response)
if dict.action == "ACK-FORWARD-PUT":
#print("Received forward PUT ack")
REQUESTS[dict.request] = True
if dict.action == "ACK-FORWARD-GET":
#print("Received forward Get ack")
REQUESTS[dict.request] = True
if dict.action == "ACK-PUT":
#print("PUT ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]])
if dict.request not in HISTORY:
HISTORY[dict.request] = set()
HISTORY[dict.request].add(PORT_TO_ID[addr[1]])
if dict.action == "ACK-GET":
#print("GET ACK received by "+self.id +" from "+ PORT_TO_ID[addr[1]])
if dict.request not in HISTORY:
HISTORY[dict.request] = list()
HISTORY[dict.request].append(
(PORT_TO_ID[addr[1]], dict.value))
if dict.action == "SYNC":
#print(self.id+"Received handoff Sync")
thread10 = Thread(target=self.synchronize, args=[dict])
thread10.start()
response = Request("SYNC-ACK", None)
Messaging.send_message(self, PORT_TO_ID[addr[1]],
response)
if (dict.action == "SYNC-ACK"):
#print(self.id + " Synced " + PORT_TO_ID[addr[1]])
self.sync_kv.pop(PORT_TO_ID[addr[1]], None)
self.check_for_sync.remove(PORT_TO_ID[addr[1]])
if (dict.action == "PONG"):
self.failed_nodes.remove(PORT_TO_ID[addr[1]])
#print("node" + str(PORT_TO_ID[addr[1]])+" is alive")
#print("Failed nodes=" + str(self.failed_nodes))
if (dict.action == "ENTROPY"):
##print(self.id+" received entropy")
threadb = Thread(target=self.antientropy, args=[dict])
threadb.start()
else:
if dict.action == "REVIVE":
#print(self.id+" is reviving")
self.state = 1
except Exception:
self.run()
def perform_antientropy(self):
for key in self.kv:
preference_list = self.hash_ring.get_node(key)
for node in preference_list:
sync_message = Request("ENTROPY", key, self.kv[key])
Messaging.send_message(self, node.id, sync_message)
