class ViewOfEntireCumulusSystem(ComputedGraph.Location):
viewOfSystem_ = ComputedGraph.Mutable(object, lambda: ())
recentGlobalUserFacingLogMessages_ = ComputedGraph.Mutable(
object, lambda: ())
totalMessageCountsEver_ = ComputedGraph.Mutable(object, lambda: 0)
@ComputedGraph.ExposedProperty()
def mostRecentMessages(self):
return self.recentGlobalUserFacingLogMessages_
@ComputedGraph.ExposedProperty()
def totalMessagesEver(self):
return self.totalMessageCountsEver_
@ComputedGraph.ExposedFunction()
def clearMostRecentMessages(self, arg):
self.recentGlobalUserFacingLogMessages_ = ()
@ComputedGraph.ExposedFunction()
def clearAndReturnMostRecentMessages(self, arg):
messages = self.recentGlobalUserFacingLogMessages_
self.recentGlobalUserFacingLogMessages_ = ()
return messages
@ComputedGraph.ExposedProperty()
def viewOfCumulusSystem(self):
return self.viewOfSystem_
@ComputedGraph.ExposedFunction()
def pushNewGlobalUserFacingLogMessage(self, msg):
self.totalMessageCountsEver_ = self.totalMessageCountsEver_ + 1
self.recentGlobalUserFacingLogMessages_ = (
self.recentGlobalUserFacingLogMessages_ + \
({"timestamp": msg.timestamp, "message": msg.message, "isDeveloperFacing": msg.isDeveloperFacing, },)
)
class PersistentCacheIndex(ComputedGraph.Location):
totalBytesInCache = ComputedGraph.Mutable(object, lambda: 0)
totalObjectsInCache = ComputedGraph.Mutable(object, lambda: 0)
totalComputationsInCache = ComputedGraph.Mutable(object, lambda: 0)
totalReachableComputationsInCache = ComputedGraph.Mutable(
object, lambda: 0)
@ComputedGraph.ExposedProperty()
def persistentCacheState(self):
return {
"totalBytesInCache":
self.totalBytesInCache,
"totalObjectsInCache":
self.totalObjectsInCache,
"totalComputationsInCache":
self.totalComputationsInCache,
"totalReachableComputationsInCache":
self.totalReachableComputationsInCache
}
@ComputedGraph.ExposedFunction()
def triggerGarbageCollectionImmediately(self, completePurge):
ComputedValueGateway.getGateway(
).triggerPerstistentCacheGarbageCollection(
True if completePurge else False)
@ComputedGraph.ExposedFunction()
def setMaxBytesInCache(self, *args):
ComputedValueGateway.getGateway().getPersistentCacheIndex(
).setMaxBytesInCache(args[0])
@ComputedGraph.ExposedProperty()
def maxBytesInCache(self):
if ComputedValueGateway.getGateway().getPersistentCacheIndex() is None:
return 0
return ComputedValueGateway.getGateway().getPersistentCacheIndex(
).getMaxBytesInCache()
@ComputedGraph.Function
def update(self):
if ComputedValueGateway.getGateway().getPersistentCacheIndex() is None:
return
self.totalBytesInCache = ComputedValueGateway.getGateway(
).getPersistentCacheIndex().totalBytesInCache()
self.totalObjectsInCache = ComputedValueGateway.getGateway(
).getPersistentCacheIndex().totalObjectsInCache()
self.totalComputationsInCache = ComputedValueGateway.getGateway(
).getPersistentCacheIndex().totalComputationsInCache()
self.totalReachableComputationsInCache = ComputedValueGateway.getGateway(
).getPersistentCacheIndex().totalReachableComputationsInCache()
class SubscriptionKeys(ComputedGraph.Location):
subscriptionKeys = ComputedGraph.Mutable(object)
def keys(self):
if self.subscriptionKeys is None:
return []
return self.subscriptionKeys
class WriteToS3Task(ComputedGraph.Location):
#the computation we want to write
computedValue = object
bucketname = object
keyname = object
#either None if it's uploading, or {'success': True/False, ['message': msg]}
successOrError = ComputedGraph.Mutable(object,
lambda: None,
exposeToProtocol=True)
@ComputedGraph.Function
def trigger(self):
if self.successOrError is not None:
return
if self.computedValue.valueIVC is None:
self.successOrError = {
'success':
False,
'message':
"Tried to trigger write before calculation was finished."
}
return
if not self.computedValue.valueIVC.isVectorOfChar():
self.successOrError = {
'success': False,
'message': "Result should have been a string."
}
return
if self.computedValue.isException:
self.successOrError = {
'success':
False,
'message':
"Result should have been a string. Got an exception instead."
}
return
def callback(result):
if result.isSuccess():
self.successOrError = {'success': True}
else:
self.successOrError = {
'success': False,
'message': str(result)
}
ComputedValueGateway.getGateway().createExternalIoTask(
CumulusNative.ExternalIoTask.WriteCharBigvecToS3(
self.computedValue.valueIVC.getVectorBigvecGuid(),
CumulusNative.S3KeyAndCredentials(self.bucketname,
self.keyname, "", "", "")),
callback)
class TestCGLocation(ComputedGraph.Location):
definition = ComputedGraph.Key(object)
aValue = ComputedGraph.Mutable(lambda: None, exposeToProtocol=True)
def sharedStateSubspace(self):
return CGSS.Node.Keyspace(keyspacePath=("public", "writeable",
"TestCGLocation")).subspace
aValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
bValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
cValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
@ComputedGraph.ExposedProperty()
def depth(self):
if isinstance(self.definition, Test):
return self.definition.depth + 1
elif isinstance(self.definition, list):
res = 0
for x in self.definition:
res += x.depth
return res
else:
return 0
@ComputedGraph.ExposedProperty()
def testCgLocation(self):
return self
@ComputedGraph.ExposedFunction()
def aFunction(self, jsonArg):
self.aValue = jsonArg
@ComputedGraph.Function
def anUnexposedFunction(self, jsonArg):
self.aValue = jsonArg
@ComputedGraph.ExposedFunction()
def testFunction(self, arg):
return arg
@ComputedGraph.ExposedFunction(wantsCallback=True)
def aFunctionExpectingCallback(self, callback, jsonArg):
def aThread():
time.sleep(1)
callback(jsonArg)
threading.Thread(target=aThread).start()
class ComputedValueHeldByHash(ComputedValue.ComputedValue):
args = None
hash = object
valueIVC_ = ComputedGraph.Mutable(object)
def valueIVC(self):
if self.valueIVC_ is None:
return ForaNative.ImplValContainer()
return self.valueIVC_
isException = object
isFinished = True
workerCount = 0
cacheloadCount = 0
computationStatistics = None
class SimpleLocation(ComputedGraph.Location):
"""A very simple computed graph location for testing"""
value = object
mutVal = ComputedGraph.Mutable(object, lambda: 0)
def property1(self):
return self.value
def property2(self):
return self.property1 * 2
def mutValTimes2(self):
return self.mutVal * 2
def mutValTimes4(self):
return self.mutValTimes2 * 2
class ComputedValue(ComputedGraph.Location):
"""Represents the description of a single Fora computation."""
# A tuple that represents the axiom-like Fora expression to compute.
# e.g. (implValForFunction, `Call, 1, 2)
args = ComputedGraph.Key(object, default = None, validator = validateComputedValueArgs)
# Result is an Interpreter::ComputationResult (defined in # ufora/FORA/Core/ComputationResult.hppml)
# which may consist of an Exception, Result, or Failure.
result = ComputedGraph.Mutable(lambda: None)
computationStatistics = ComputedGraph.Mutable(lambda: None)
def __hash__(self):
return hash(self.hash)
def __cmp__(self, other):
self.depth
return TypeAwareComparison.typecmp(self, other,
lambda self, other : cmp(self.hash, other.hash))
def depth(self):
"""compute the depth of the ComputedValue tree"""
tr = 0
if self.args is None:
return 0
for a in self.args:
try:
tr = max(a.depth + 1, tr)
except AttributeError:
pass
assert tr < 50, self
return tr
@ComputedGraph.ExposedProperty()
def asVector(self):
if not self.isFinished:
return None
if self.isException:
return None
if not self.valueIVC.isVector():
return None
return ComputedValueVectorFromComputedValue(computedValue = self)
def computationDefinitionTerm_(self):
return CumulusNative.ComputationDefinitionTerm.Subcomputation(
self.cumulusComputationDefinition.asRoot.terms
)
def cumulusComputationDefinition(self):
terms = []
assert self.args is not None, self.__location_class__
for a in self.args:
if isinstance(a, (long, int, str, bool)):
terms.append(CumulusNative.ComputationDefinitionTerm.Value(ImplValContainer_(a), None))
elif isinstance(a, ImplValContainer_):
terms.append(CumulusNative.ComputationDefinitionTerm.Value(a, None))
else:
terms.append(a.computationDefinitionTerm_)
return CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(terms)
)
@ComputedGraph.ExposedProperty()
def submittedComputationId(self):
computationId = ComputedValueGateway.getGateway().submittedComputationId(self.cumulusComputationDefinition)
if computationId is None:
return
return computationId.toSimple()
@ComputedGraph.ExposedFunction()
def cancel(self, *args):
ComputedValueGateway.getGateway().cancelComputation(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def requestComputationCheckpoint(self, *args):
ComputedValueGateway.getGateway().requestComputationCheckpoint(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def increaseRequestCount(self, *args):
ComputedValueGateway.getGateway().increaseRequestCount(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def decreaseRequestCount(self, *args):
ComputedValueGateway.getGateway().decreaseRequestCount(self, self.cumulusComputationDefinition)
def totalVectorBytesReferenced(self):
if self.checkpointStatus is None:
return 0
stats = self.checkpointStatus.statistics
return ComputedValueGateway.getGateway().bytecountForBigvecs(
self.checkpointStatus.bigvecsReferenced
)
def totalBytesOfMemoryReferenced(self):
if self.checkpointStatus is None:
return 0
stats = self.checkpointStatus.statistics
return stats.totalBytesInMemory
def isCompletelyCheckpointed(self):
if self.checkpointStatus is None:
return False
stats = self.checkpointStatus.statistics
totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter
return self.totalComputeSecondsAtLastCheckpoint + 1.0 > totalSeconds
@ComputedGraph.ExposedProperty()
def unfinishedDependentCodeLocationsAsJson(self):
res = []
for cv in self.rootComputedValueDependencies:
if not cv.isFinished:
#each computed value we depend on might be a visualize. If so, we want
#the actual script computation (if available)
cv = cv.unwrapVisualizable
cv = cv.unwrapMember
#only some ComputedValue objects have this property
loc = cv.codeLocationDefinitionAsJson
if loc is not None:
res.append(loc)
else:
res.append(str(ComputedGraph.getLocationTypeFromLocation(cv)))
return tuple(res)
@ComputedGraph.ExposedProperty()
def stats(self):
if self.checkpointStatus is None:
return {}
stats = self.checkpointStatus.statistics
result = {
"status": {
"title" : "Computation Status",
"value" : "Finished" if self.isFinished else "Unfinished" +
((" (%s cpus)" % self.totalWorkerCount) if self.totalWorkerCount > 0 else ""),
"units" : ""
},
"timeSpentInCompiler": {
"title" : "Time in compiled code (across all cores)",
"value" : stats.timeSpentInCompiler,
"units" : "sec"
},
"timeSpentInInterpreter": {
"title" : "Time in interpreted code (across all cores)",
"value" : stats.timeSpentInInterpreter,
"units" : "sec"
},
"totalSplitCount": {
"title" : "Total split count",
"value" : stats.totalSplitCount,
"units" : ""
},
"totalBytesReferenced": {
"title" : "Total bytes referenced (calculations)",
"value" : self.totalBytesOfMemoryReferenced,
"units" : "bytes"
},
"totalBytesReferencedJustPaged": {
"title" : "Total bytes referenced (vectors)",
"value" : self.totalVectorBytesReferenced,
"units" : "bytes"
}
}
result["isCheckpointing"] = {
"title" : "Is Checkpointing",
"value" : self.isCheckpointing,
"units" : ""
}
result["isLoadingFromCheckpoint"] = {
"title" : "Is loading from checkpoint",
"value" : self.isLoadingFromCheckpoint,
"units" : ""
}
if self.totalBytesReferencedAtLastCheckpoint > 0:
result["totalBytesReferencedAtLastCheckpoint"] = {
'title': "Size of last checkpoint",
'value': self.totalBytesReferencedAtLastCheckpoint,
'units': 'bytes'
}
secondsAtCheckpoint = self.totalComputeSecondsAtLastCheckpoint
if secondsAtCheckpoint == 0.0:
result["checkpointStatus"] = {
"title" : "Checkpoint Status",
"value" : "not checkpointed",
"units" : ""
}
else:
totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter
if secondsAtCheckpoint + 1.0 >= totalSeconds:
result["checkpointStatus"] = {
"title" : "Checkpoint Status",
"value" : "Checkpointed",
"units" : ""
}
else:
result["checkpointStatus"] = {
"title" : "Uncheckpointed compute seconds",
"value" : totalSeconds - secondsAtCheckpoint,
"units" : "sec"
}
return result
def isFailure(self):
if not self.result:
return None
return self.result.isFailure()
def failure(self):
if not self.result:
return None
return self.result.asFailure.error.toString()
@ComputedGraph.ExposedProperty()
def isException(self):
if not self.result:
return None
return self.result.isException()
def valueIVC(self):
if not self.result:
return None
if self.result.isException():
return self.result.asException.exception
if self.result.isResult():
return self.result.asResult.result
return None
@ComputedGraph.ExposedProperty()
def isFinished(self):
return self.valueIVC is not None or self.isFailure
def isEmpty(self):
if not self.isFinished:
return True
if self.isException:
return False
if self.isFailure:
return False
if self.valueIVC is None:
return True
if self.isVector:
return self.valueIVC.getVectorSize() == 0
if self.valueIVC.isNothing():
return True
if self.valueIVC.isString():
return len(self.valueIVC.pyval) == 0
if self.valueIVC.isTuple():
return len(self.valueIVC) == 0
return False
workerCount = ComputedGraph.Mutable(object, lambda: 0)
workerCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0)
cacheloadCount = ComputedGraph.Mutable(object, lambda: 0)
cacheloadCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0)
checkpointStatus = ComputedGraph.Mutable(object, lambda: None)
totalComputeSecondsAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0.0)
isCheckpointing = ComputedGraph.Mutable(object, lambda: False)
isLoadingFromCheckpoint = ComputedGraph.Mutable(object, lambda: False)
rootComputedValueDependencies = ComputedGraph.Mutable(object, lambda: ())
totalBytesReferencedAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0)
@ComputedGraph.ExposedProperty()
def totalWorkerCount(self):
return self.workerCount + self.workerCountForDependentComputations
def codeLocationDefinitionAsJson(self):
return None
@ComputedGraph.ExposedProperty()
def totalCacheloadCount(self):
return self.cacheloadCount + self.cacheloadCountForDependentComputations
def __str__(self):
if self.args is None:
return "ComputedValue(None)"
return "ComputedValue" + str(tuple(self.args))
def hash(self):
h0 = self.hashValue_(self.args[0])
for arg in self.args[1:]:
h0 = h0 + self.hashValue_(arg)
return h0
@ComputedGraph.Function
def hashValue_(self, value):
if hasattr(value, 'hash'):
return value.hash
logging.debug("Using python hash on type '%s'. %s", type(value), str(value))
hashValue = ctypes.c_uint32(hash(value)).value
return Hash.Hash(hashValue)
def isVector(self):
if self.isFailure:
return False
if self.valueIVC is None:
return False
return self.valueIVC.isVector()
@ComputedGraph.ExposedFunction(expandArgs=True)
def writeToS3(self, bucketname, keyname):
"""Trigger a task writing this dataset to amazon S3.
Returns a WriteToS3Task object.
"""
if not isinstance(bucketname, str):
raise Exceptions.SubscribableWebObjectsException("Expected bucketname to be a string")
if not isinstance(keyname, str):
raise Exceptions.SubscribableWebObjectsException("Expected keyname to be a string")
if self.valueIVC is None:
return None
task = WriteToS3Task.WriteToS3Task(computedValue=self, bucketname=bucketname, keyname=keyname)
task.trigger()
return task
class ComputedValueVectorSlice(ComputedGraph.Location):
computedValueVector = object
lowIndex = object
highIndex = object
isLoaded = ComputedGraph.Mutable(lambda: False)
@ComputedGraph.NotCached
def vectorDataIds(self):
if self.computedValueVector.vectorImplVal is None:
return []
return self.computedValueVector.vectorImplVal.getVectorDataIdsForSlice(
self.lowIndex,
self.highIndex,
ComputedValueGateway.getGateway().vdm
)
def __str__(self):
return "ComputedValueVectorSlice(%s, %s, %s)" % (
self.computedValueVector,
self.lowIndex,
self.highIndex
)
@ComputedGraph.Function
def extractVectorDataAsPythonArray(self):
if self.computedValueVector.vectorImplVal is None:
return None
if len(self.vectorDataIds) > 0 and not self.isLoaded:
return None
result = ComputedValueGateway.getGateway().extractVectorDataAsPythonArray(
self.computedValueVector,
self.lowIndex,
self.highIndex
)
if result is None and not self.vdmThinksIsLoaded():
logging.info("CumulusClient: %s was marked loaded but returned None. reloading", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.Function
def vdmThinksIsLoaded(self):
return ComputedValueGateway.getGateway().vectorDataIsLoaded(
self.computedValueVector,
self.lowIndex,
self.highIndex
)
@ComputedGraph.Function
def markLoaded(self, isLoaded):
self.isLoaded = isLoaded
@ComputedGraph.Function
def extractVectorDataAsNumpyArray(self):
logging.info("Extract numpy data for %s: %s", self, self.vdmThinksIsLoaded())
if self.computedValueVector.vectorImplVal is None:
return None
if len(self.vectorDataIds) > 0 and not self.isLoaded:
return None
result = ComputedValueGateway.getGateway().extractVectorDataAsNumpyArray(
self.computedValueVector,
self.lowIndex,
self.highIndex
)
if result is None and not self.vdmThinksIsLoaded():
logging.info("CumulusClient: %s was marked loaded but returned None", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.Function
def extractVectorDataAsNumpyArrayInChunks(self, stepSize = 100000):
"""Return the data as a sequence of numpy arrays each of which is no larger than 'stepSize'.
This is used to prevent us from creating memory fragmentation when we are loading
lots of arrays of different sizes.
"""
logging.info("Extract numpy data for %s: %s", self, self.vdmThinksIsLoaded())
if self.computedValueVector.vectorImplVal is None:
return None
if len(self.vectorDataIds) > 0 and not self.isLoaded:
return None
if not self.vdmThinksIsLoaded():
return None
result = []
index = self.lowIndex
while index < self.highIndex and result is not None:
tailResult = ComputedValueGateway.getGateway().extractVectorDataAsNumpyArray(
self.computedValueVector,
index,
min(self.highIndex, index+stepSize)
)
index += stepSize
if tailResult is not None:
result.append(tailResult)
else:
result = None
if result is None and not self.vdmThinksIsLoaded():
logging.info("CumulusClient: %s was marked loaded but returned None", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.Function
def extractVectorItemAsIVC(self, ct):
if self.computedValueVector.vectorImplVal is None:
return None
if len(self.vectorDataIds) > 0 and not self.isLoaded:
return None
result = ComputedValueGateway.getGateway().extractVectorItem(self.computedValueVector, ct)
if result is None:
logging.info("CumulusClient: %s was marked loaded but returned None", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.ExposedFunction()
def getSlice(self, lowIndex, highIndex):
assert lowIndex >= 0
assert lowIndex < highIndex
assert highIndex <= self.highIndex - self.lowIndex
return ComputedValueVectorSlice(
computedValueVector = self.computedValueVector,
lowIndex = self.lowIndex + lowIndex,
highIndex = self.lowIndex + highIndex
)
@ComputedGraph.ExposedFunction()
def increaseRequestCount(self, *args):
"""request the data in the leaf of this vector"""
ComputedValueGateway.getGateway().increaseVectorRequestCount(self)
@ComputedGraph.ExposedFunction()
def decreaseRequestCount(self, *args):
ComputedValueGateway.getGateway().decreaseVectorRequestCount(self)
class Keyspace(ComputedGraph.Location):
keyspacePath = ComputedGraph.Key(object, validator=isTuple)
isLoaded_ = ComputedGraph.Mutable(object, lambda: False)
toCallOnLoad_ = ComputedGraph.Mutable(object, lambda: ())
isSubscribed_ = ComputedGraph.Mutable(object, lambda: False)
@ComputedGraph.Function
def onLoad(self, toCallOnLoad):
if self.isLoaded_:
toCallOnLoad()
else:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.waitLoaded()
toCallOnLoad()
return
else:
self.toCallOnLoad_ = self.toCallOnLoad_ + (toCallOnLoad, )
self.ensureSubscribed()
def keyspaceName(self):
return keyspacePathToKeyspaceName(self.keyspacePath)
@ComputedGraph.Function
def __str__(self):
return "Keyspace(%s)" % (self.keyspacePath, )
@ComputedGraph.Function
def ensureSubscribed(self):
assert SharedStateSynchronizer.isConnected()
if not self.isSubscribed_:
logging.info("ComputedGraphSharedState subscribing to: %s", self)
SharedStateSynchronizer.getSynchronizer().addKeyspaceListener(
self.keyspaceName, KeyspaceUpdater(self), NativeJson.Json(()))
self.isSubscribed_ = True
@ComputedGraph.Function
def markLoaded(self):
if self.isLoaded_:
return
logging.info("ComputedGraphSharedState marking loaded: %s", self)
self.isLoaded_ = True
self.ensureSubscribed()
for toCallOnLoad in self.toCallOnLoad_:
toCallOnLoad()
self.toCallOnLoad_ = ()
@ComputedGraph.Function
def waitLoaded(self):
if not self.isLoaded_:
self.ensureSubscribed()
SharedStateSynchronizer.getSynchronizer().waitForKeyspaceAndPrefix(
self.keyspaceName, NativeJson.Json(()))
def loaded(self):
if (not self.isLoaded_ and SynchronousPropertyAccess.
SynchronousPropertyAccess.getCurrent() is not None):
self.ensureSubscribed()
self.waitLoaded()
return True
else:
self.ensureSubscribed()
return self.isLoaded_
def subspace(self):
return Subspace(keyspace=self, keyPath=())
@ComputedGraph.Function
def subKeyspace(self, subspaceName):
return Keyspace(keyspacePath=self.keyspacePath + (subspaceName, ))
@ComputedGraph.Function
def assertLoaded(self):
if not self.loaded:
raise NotLoadedException(self)
class Subspace(ComputedGraph.Location):
keyspace = object
keyPath = ComputedGraph.Key(object, validator=isTuple)
value_ = ComputedGraph.Mutable(object, lambda: None)
def keyName(self):
return NativeJson.Json(('CGSS', JsonPickle.toJson(self.keyPath)))
def loaded(self):
return self.keyspace.loaded
@ComputedGraph.Function
def subKeyspace(self, subKeyspaceName):
return Subspace(keyspace=self.keyspace.subKeyspace(subKeyspaceName),
keyPath=self.keyPath)
@ComputedGraph.Function
def subspace(self, subspace):
return Subspace(keyspace=self.keyspace,
keyPath=self.keyPath + (subspace, ))
@ComputedGraph.Function
def __str__(self):
return "Subspace(%s,%s)" % (self.keyspace, self.keyPath)
@ComputedGraph.Function
def setValueSlot(self, newValue):
logging.info("Setting %s %s", str(self), newValue)
if not self.keyspace.loaded:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.keyspace.waitLoaded()
else:
logging.info("raising NotLoadedException for %s", self)
raise NotLoadedException(self.keyspace)
if newValue is None:
SharedStateSynchronizer.getSynchronizer().writeValue(
self.keyspace.keyspaceName, self.keyName, None)
else:
assert isinstance(newValue, tuple)
assert len(newValue) == 1
try:
SharedStateSynchronizer.getSynchronizer().writeValue(
self.keyspace.keyspaceName, self.keyName,
JsonPickle.toJson(newValue[0]))
except Exception as e:
logging.error("%s", traceback.format_exc())
raise
self.value_ = newValue
@ComputedGraph.WithSetter(setValueSlot)
def value(self):
if not self.keyspace.loaded:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.keyspace.waitLoaded()
else:
logging.info("raising NotLoadedException for %s", self)
raise NotLoadedException(self.keyspace)
return self.value_
class ComputedValueVectorSlice(ComputedGraph.Location):
computedValueVector = object
lowIndex = object
highIndex = object
isLoaded = ComputedGraph.Mutable(lambda: False)
@ComputedGraph.NotCached
def vectorDataIds(self):
if self.computedValueVector.vectorImplVal is None:
return []
return self.computedValueVector.vectorImplVal.getVectorDataIdsForSlice(
self.lowIndex, self.highIndex,
ComputedValueGateway.getGateway().vdm)
def __str__(self):
return "ComputedValueVectorSlice(%s, %s, %s)" % (
self.computedValueVector, self.lowIndex, self.highIndex)
@ComputedGraph.Function
def extractVectorDataAsPythonArray(self):
if self.computedValueVector.vectorImplVal is None:
return None
if len(self.vectorDataIds) > 0 and not self.isLoaded:
return None
result = ComputedValueGateway.getGateway(
).extractVectorDataAsPythonArray(self.computedValueVector,
self.lowIndex, self.highIndex)
if result is None:
logging.warn(
"CumulusClient: %s was marked loaded but returned None. reloading",
self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.Function
def extractVectorDataAsNumpyArray(self):
if self.computedValueVector.vectorImplVal is None:
return None
if not self.isLoaded:
return None
result = ComputedValueGateway.getGateway(
).extractVectorDataAsNumpyArray(self.computedValueVector,
self.lowIndex, self.highIndex)
if result is None:
logging.warn(
"CumulusClient: %s was marked loaded but returned None", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.Function
def extractVectorItemAsIVC(self, ct):
if self.computedValueVector.vectorImplVal is None:
return None
if not self.isLoaded:
return None
result = ComputedValueGateway.getGateway().extractVectorItem(
self.computedValueVector, ct)
if result is None:
logging.warn(
"CumulusClient: %s was marked loaded but returned None", self)
self.isLoaded = False
ComputedValueGateway.getGateway().reloadVector(self)
return result
@ComputedGraph.ExposedFunction()
def getSlice(self, lowIndex, highIndex):
assert lowIndex >= 0
assert lowIndex < highIndex
assert highIndex <= self.highIndex - self.lowIndex
return ComputedValueVectorSlice(
computedValueVector=self.computedValueVector,
lowIndex=self.lowIndex + lowIndex,
highIndex=self.lowIndex + highIndex)
@ComputedGraph.ExposedFunction()
def increaseRequestCount(self, *args):
"""request the data in the leaf of this vector"""
ComputedValueGateway.getGateway().increaseVectorRequestCount(self)
@ComputedGraph.ExposedFunction()
def decreaseRequestCount(self, *args):
ComputedValueGateway.getGateway().decreaseVectorRequestCount(self)