class PyforaTupleElement(PyforaComputedValue):
#the base PyforaComputedValue of which we are one of the tuple members
baseCV = ComputedGraph.Key(object)
#the index in self.baseCV.pyforaTupleToTuple we are tied to
index = ComputedGraph.Key(object)
def argIds(self):
return ()
def args(self):
return (self.baseCV,
ForaNative.makeSymbol("RawGetItemByInt"),
ForaNative.ImplValContainer(self.index))
def __str__(self):
return "PyforaTupleElement(baseCV=%s,index=%s)" % (self.baseCV, self.index)
class PyforaDictionaryElement(PyforaComputedValue):
#the base PyforaComputedValue of which we are one of the dictionary members
baseCV = ComputedGraph.Key(object)
#the keyname in self.baseCV.pyforaDictToStringDict we are tied to
keyname = ComputedGraph.Key(object)
def argIds(self):
return ()
def args(self):
return (self.baseCV,
ForaNative.makeSymbol("RawGetItemByString"),
ForaNative.ImplValContainer(self.keyname))
def __str__(self):
return "PyforaDictionaryElement(baseCV=%s,keyname=%s)" % (self.baseCV, self.keyname)
class TestCGLocation(ComputedGraph.Location):
definition = ComputedGraph.Key(object)
aValue = ComputedGraph.Mutable(lambda: None, exposeToProtocol=True)
def sharedStateSubspace(self):
return CGSS.Node.Keyspace(keyspacePath=("public", "writeable",
"TestCGLocation")).subspace
aValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
bValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
cValueInSharedState = CGSS.Property.Property(default=lambda: False,
exposeToProtocol=True)
@ComputedGraph.ExposedProperty()
def depth(self):
if isinstance(self.definition, Test):
return self.definition.depth + 1
elif isinstance(self.definition, list):
res = 0
for x in self.definition:
res += x.depth
return res
else:
return 0
@ComputedGraph.ExposedProperty()
def testCgLocation(self):
return self
@ComputedGraph.ExposedFunction()
def aFunction(self, jsonArg):
self.aValue = jsonArg
@ComputedGraph.Function
def anUnexposedFunction(self, jsonArg):
self.aValue = jsonArg
@ComputedGraph.ExposedFunction()
def testFunction(self, arg):
return arg
@ComputedGraph.ExposedFunction(wantsCallback=True)
def aFunctionExpectingCallback(self, callback, jsonArg):
def aThread():
time.sleep(1)
callback(jsonArg)
threading.Thread(target=aThread).start()
class ComputedValueVector(ComputedGraph.Location):
"""A Location representing a loadable piece of data in the Gateway's RAM cache.
Incrementing its request count causes it to try to be loaded, and decrementing
it gets rid of the dependency.
Clients can access the data as a numpy array or pull out individual FORA values.
"""
vectorImplVal = ComputedGraph.Key(object, default = None, validator = validateIsVectorIVC)
@ComputedGraph.Function
def __getitem__(self, index):
"""if index is an integer, attempt to index into the vector.
If index is a slice, produce a computed value with the sliced vector
"""
if self.vectorImplVal is None:
raise IndexError()
if isinstance(index, slice):
return ComputedValue(
args = (
self.vectorImplVal,
ForaNative.makeSymbol("GetItemDeepcopied"),
ForaNative.ImplValContainer(index.start),
ForaNative.ImplValContainer(index.stop),
ForaNative.ImplValContainer(index.step)
)
)
if index < 0 or index >= self.elementCount:
raise IndexError()
return self.entireSlice.extractVectorItemAsIVC(index)
@ComputedGraph.ExposedProperty()
def elementCount(self):
if self.vectorImplVal is None:
return 0
return self.vectorImplVal.getVectorSize()
def entireSlice(self):
return self.getMappableSlice(0, self.elementCount)
def slicesByPage(self):
if self.vectorImplVal is None:
return []
return [self.getMappableSlice(low,high) for low,high in
self.vectorImplVal.getVectorPageSliceRanges(
ComputedValueGateway.getGateway().vdm
)]
@ComputedGraph.Function
def getMappableSlice(self, lowIndex, highIndex):
return ComputedValueVectorSlice(
computedValueVector = self,
lowIndex = lowIndex,
highIndex = highIndex
)
@ComputedGraph.ExposedFunction(expandArgs=True)
def getSlice(self, lowIndex, highIndex):
return self.getMappableSlice(lowIndex, highIndex)
def __str__(self):
return "%s" % (self.vectorImplVal)
class ComputedValue(ComputedGraph.Location):
"""Represents the description of a single Fora computation."""
# A tuple that represents the axiom-like Fora expression to compute.
# e.g. (implValForFunction, `Call, 1, 2)
args = ComputedGraph.Key(object, default = None, validator = validateComputedValueArgs)
# Result is an Interpreter::ComputationResult (defined in # ufora/FORA/Core/ComputationResult.hppml)
# which may consist of an Exception, Result, or Failure.
result = ComputedGraph.Mutable(lambda: None)
computationStatistics = ComputedGraph.Mutable(lambda: None)
def __hash__(self):
return hash(self.hash)
def __cmp__(self, other):
self.depth
return TypeAwareComparison.typecmp(self, other,
lambda self, other : cmp(self.hash, other.hash))
def depth(self):
"""compute the depth of the ComputedValue tree"""
tr = 0
if self.args is None:
return 0
for a in self.args:
try:
tr = max(a.depth + 1, tr)
except AttributeError:
pass
assert tr < 50, self
return tr
@ComputedGraph.ExposedProperty()
def asVector(self):
if not self.isFinished:
return None
if self.isException:
return None
if not self.valueIVC.isVector():
return None
return ComputedValueVectorFromComputedValue(computedValue = self)
def computationDefinitionTerm_(self):
return CumulusNative.ComputationDefinitionTerm.Subcomputation(
self.cumulusComputationDefinition.asRoot.terms
)
def cumulusComputationDefinition(self):
terms = []
assert self.args is not None, self.__location_class__
for a in self.args:
if isinstance(a, (long, int, str, bool)):
terms.append(CumulusNative.ComputationDefinitionTerm.Value(ImplValContainer_(a), None))
elif isinstance(a, ImplValContainer_):
terms.append(CumulusNative.ComputationDefinitionTerm.Value(a, None))
else:
terms.append(a.computationDefinitionTerm_)
return CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(terms)
)
@ComputedGraph.ExposedProperty()
def submittedComputationId(self):
computationId = ComputedValueGateway.getGateway().submittedComputationId(self.cumulusComputationDefinition)
if computationId is None:
return
return computationId.toSimple()
@ComputedGraph.ExposedFunction()
def cancel(self, *args):
ComputedValueGateway.getGateway().cancelComputation(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def requestComputationCheckpoint(self, *args):
ComputedValueGateway.getGateway().requestComputationCheckpoint(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def increaseRequestCount(self, *args):
ComputedValueGateway.getGateway().increaseRequestCount(self, self.cumulusComputationDefinition)
@ComputedGraph.ExposedFunction()
def decreaseRequestCount(self, *args):
ComputedValueGateway.getGateway().decreaseRequestCount(self, self.cumulusComputationDefinition)
def totalVectorBytesReferenced(self):
if self.checkpointStatus is None:
return 0
stats = self.checkpointStatus.statistics
return ComputedValueGateway.getGateway().bytecountForBigvecs(
self.checkpointStatus.bigvecsReferenced
)
def totalBytesOfMemoryReferenced(self):
if self.checkpointStatus is None:
return 0
stats = self.checkpointStatus.statistics
return stats.totalBytesInMemory
def isCompletelyCheckpointed(self):
if self.checkpointStatus is None:
return False
stats = self.checkpointStatus.statistics
totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter
return self.totalComputeSecondsAtLastCheckpoint + 1.0 > totalSeconds
@ComputedGraph.ExposedProperty()
def unfinishedDependentCodeLocationsAsJson(self):
res = []
for cv in self.rootComputedValueDependencies:
if not cv.isFinished:
#each computed value we depend on might be a visualize. If so, we want
#the actual script computation (if available)
cv = cv.unwrapVisualizable
cv = cv.unwrapMember
#only some ComputedValue objects have this property
loc = cv.codeLocationDefinitionAsJson
if loc is not None:
res.append(loc)
else:
res.append(str(ComputedGraph.getLocationTypeFromLocation(cv)))
return tuple(res)
@ComputedGraph.ExposedProperty()
def stats(self):
if self.checkpointStatus is None:
return {}
stats = self.checkpointStatus.statistics
result = {
"status": {
"title" : "Computation Status",
"value" : "Finished" if self.isFinished else "Unfinished" +
((" (%s cpus)" % self.totalWorkerCount) if self.totalWorkerCount > 0 else ""),
"units" : ""
},
"timeSpentInCompiler": {
"title" : "Time in compiled code (across all cores)",
"value" : stats.timeSpentInCompiler,
"units" : "sec"
},
"timeSpentInInterpreter": {
"title" : "Time in interpreted code (across all cores)",
"value" : stats.timeSpentInInterpreter,
"units" : "sec"
},
"totalSplitCount": {
"title" : "Total split count",
"value" : stats.totalSplitCount,
"units" : ""
},
"totalBytesReferenced": {
"title" : "Total bytes referenced (calculations)",
"value" : self.totalBytesOfMemoryReferenced,
"units" : "bytes"
},
"totalBytesReferencedJustPaged": {
"title" : "Total bytes referenced (vectors)",
"value" : self.totalVectorBytesReferenced,
"units" : "bytes"
}
}
result["isCheckpointing"] = {
"title" : "Is Checkpointing",
"value" : self.isCheckpointing,
"units" : ""
}
result["isLoadingFromCheckpoint"] = {
"title" : "Is loading from checkpoint",
"value" : self.isLoadingFromCheckpoint,
"units" : ""
}
if self.totalBytesReferencedAtLastCheckpoint > 0:
result["totalBytesReferencedAtLastCheckpoint"] = {
'title': "Size of last checkpoint",
'value': self.totalBytesReferencedAtLastCheckpoint,
'units': 'bytes'
}
secondsAtCheckpoint = self.totalComputeSecondsAtLastCheckpoint
if secondsAtCheckpoint == 0.0:
result["checkpointStatus"] = {
"title" : "Checkpoint Status",
"value" : "not checkpointed",
"units" : ""
}
else:
totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter
if secondsAtCheckpoint + 1.0 >= totalSeconds:
result["checkpointStatus"] = {
"title" : "Checkpoint Status",
"value" : "Checkpointed",
"units" : ""
}
else:
result["checkpointStatus"] = {
"title" : "Uncheckpointed compute seconds",
"value" : totalSeconds - secondsAtCheckpoint,
"units" : "sec"
}
return result
def isFailure(self):
if not self.result:
return None
return self.result.isFailure()
def failure(self):
if not self.result:
return None
return self.result.asFailure.error.toString()
@ComputedGraph.ExposedProperty()
def isException(self):
if not self.result:
return None
return self.result.isException()
def valueIVC(self):
if not self.result:
return None
if self.result.isException():
return self.result.asException.exception
if self.result.isResult():
return self.result.asResult.result
return None
@ComputedGraph.ExposedProperty()
def isFinished(self):
return self.valueIVC is not None or self.isFailure
def isEmpty(self):
if not self.isFinished:
return True
if self.isException:
return False
if self.isFailure:
return False
if self.valueIVC is None:
return True
if self.isVector:
return self.valueIVC.getVectorSize() == 0
if self.valueIVC.isNothing():
return True
if self.valueIVC.isString():
return len(self.valueIVC.pyval) == 0
if self.valueIVC.isTuple():
return len(self.valueIVC) == 0
return False
workerCount = ComputedGraph.Mutable(object, lambda: 0)
workerCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0)
cacheloadCount = ComputedGraph.Mutable(object, lambda: 0)
cacheloadCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0)
checkpointStatus = ComputedGraph.Mutable(object, lambda: None)
totalComputeSecondsAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0.0)
isCheckpointing = ComputedGraph.Mutable(object, lambda: False)
isLoadingFromCheckpoint = ComputedGraph.Mutable(object, lambda: False)
rootComputedValueDependencies = ComputedGraph.Mutable(object, lambda: ())
totalBytesReferencedAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0)
@ComputedGraph.ExposedProperty()
def totalWorkerCount(self):
return self.workerCount + self.workerCountForDependentComputations
def codeLocationDefinitionAsJson(self):
return None
@ComputedGraph.ExposedProperty()
def totalCacheloadCount(self):
return self.cacheloadCount + self.cacheloadCountForDependentComputations
def __str__(self):
if self.args is None:
return "ComputedValue(None)"
return "ComputedValue" + str(tuple(self.args))
def hash(self):
h0 = self.hashValue_(self.args[0])
for arg in self.args[1:]:
h0 = h0 + self.hashValue_(arg)
return h0
@ComputedGraph.Function
def hashValue_(self, value):
if hasattr(value, 'hash'):
return value.hash
logging.debug("Using python hash on type '%s'. %s", type(value), str(value))
hashValue = ctypes.c_uint32(hash(value)).value
return Hash.Hash(hashValue)
def isVector(self):
if self.isFailure:
return False
if self.valueIVC is None:
return False
return self.valueIVC.isVector()
@ComputedGraph.ExposedFunction(expandArgs=True)
def writeToS3(self, bucketname, keyname):
"""Trigger a task writing this dataset to amazon S3.
Returns a WriteToS3Task object.
"""
if not isinstance(bucketname, str):
raise Exceptions.SubscribableWebObjectsException("Expected bucketname to be a string")
if not isinstance(keyname, str):
raise Exceptions.SubscribableWebObjectsException("Expected keyname to be a string")
if self.valueIVC is None:
return None
task = WriteToS3Task.WriteToS3Task(computedValue=self, bucketname=bucketname, keyname=keyname)
task.trigger()
return task
class ComputedValueVectorFromComputedValue(ComputedValueVector):
computedValue = ComputedGraph.Key(object)
def vectorImplVal(self):
return self.computedValue.valueIVC
class Keyspace(ComputedGraph.Location):
keyspacePath = ComputedGraph.Key(object, validator=isTuple)
isLoaded_ = ComputedGraph.Mutable(object, lambda: False)
toCallOnLoad_ = ComputedGraph.Mutable(object, lambda: ())
isSubscribed_ = ComputedGraph.Mutable(object, lambda: False)
@ComputedGraph.Function
def onLoad(self, toCallOnLoad):
if self.isLoaded_:
toCallOnLoad()
else:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.waitLoaded()
toCallOnLoad()
return
else:
self.toCallOnLoad_ = self.toCallOnLoad_ + (toCallOnLoad, )
self.ensureSubscribed()
def keyspaceName(self):
return keyspacePathToKeyspaceName(self.keyspacePath)
@ComputedGraph.Function
def __str__(self):
return "Keyspace(%s)" % (self.keyspacePath, )
@ComputedGraph.Function
def ensureSubscribed(self):
assert SharedStateSynchronizer.isConnected()
if not self.isSubscribed_:
logging.info("ComputedGraphSharedState subscribing to: %s", self)
SharedStateSynchronizer.getSynchronizer().addKeyspaceListener(
self.keyspaceName, KeyspaceUpdater(self), NativeJson.Json(()))
self.isSubscribed_ = True
@ComputedGraph.Function
def markLoaded(self):
if self.isLoaded_:
return
logging.info("ComputedGraphSharedState marking loaded: %s", self)
self.isLoaded_ = True
self.ensureSubscribed()
for toCallOnLoad in self.toCallOnLoad_:
toCallOnLoad()
self.toCallOnLoad_ = ()
@ComputedGraph.Function
def waitLoaded(self):
if not self.isLoaded_:
self.ensureSubscribed()
SharedStateSynchronizer.getSynchronizer().waitForKeyspaceAndPrefix(
self.keyspaceName, NativeJson.Json(()))
def loaded(self):
if (not self.isLoaded_ and SynchronousPropertyAccess.
SynchronousPropertyAccess.getCurrent() is not None):
self.ensureSubscribed()
self.waitLoaded()
return True
else:
self.ensureSubscribed()
return self.isLoaded_
def subspace(self):
return Subspace(keyspace=self, keyPath=())
@ComputedGraph.Function
def subKeyspace(self, subspaceName):
return Keyspace(keyspacePath=self.keyspacePath + (subspaceName, ))
@ComputedGraph.Function
def assertLoaded(self):
if not self.loaded:
raise NotLoadedException(self)
class Subspace(ComputedGraph.Location):
keyspace = object
keyPath = ComputedGraph.Key(object, validator=isTuple)
value_ = ComputedGraph.Mutable(object, lambda: None)
def keyName(self):
return NativeJson.Json(('CGSS', JsonPickle.toJson(self.keyPath)))
def loaded(self):
return self.keyspace.loaded
@ComputedGraph.Function
def subKeyspace(self, subKeyspaceName):
return Subspace(keyspace=self.keyspace.subKeyspace(subKeyspaceName),
keyPath=self.keyPath)
@ComputedGraph.Function
def subspace(self, subspace):
return Subspace(keyspace=self.keyspace,
keyPath=self.keyPath + (subspace, ))
@ComputedGraph.Function
def __str__(self):
return "Subspace(%s,%s)" % (self.keyspace, self.keyPath)
@ComputedGraph.Function
def setValueSlot(self, newValue):
logging.info("Setting %s %s", str(self), newValue)
if not self.keyspace.loaded:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.keyspace.waitLoaded()
else:
logging.info("raising NotLoadedException for %s", self)
raise NotLoadedException(self.keyspace)
if newValue is None:
SharedStateSynchronizer.getSynchronizer().writeValue(
self.keyspace.keyspaceName, self.keyName, None)
else:
assert isinstance(newValue, tuple)
assert len(newValue) == 1
try:
SharedStateSynchronizer.getSynchronizer().writeValue(
self.keyspace.keyspaceName, self.keyName,
JsonPickle.toJson(newValue[0]))
except Exception as e:
logging.error("%s", traceback.format_exc())
raise
self.value_ = newValue
@ComputedGraph.WithSetter(setValueSlot)
def value(self):
if not self.keyspace.loaded:
if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent(
) is not None:
self.keyspace.waitLoaded()
else:
logging.info("raising NotLoadedException for %s", self)
raise NotLoadedException(self.keyspace)
return self.value_
class PyforaComputedValue(ComputedValue.ComputedValue):
argIds = ComputedGraph.Key(object, default=None, validator=validateObjectIds)
def args(self):
converter = PyforaObjectConverter.PyforaObjectConverter()
def unwrapArg(argId):
if isinstance(argId, int):
return converter.getIvcFromObjectId(argId)
else:
return argId
implVals = tuple(unwrapArg(arg) for arg in self.argIds)
return implVals[:1] + (ForaValue.FORAValue.symbol_Call.implVal_,) + implVals[1:]
def __str__(self):
return "PyforaComputedValue" + str(tuple(self.argIds))
def pyforaDictToDictOfAssignedVarsToProxyValues(self):
if self.valueIVC is None:
return None
assert not self.isException, "We should not allow exceptions to be thrown here. Instead we should " +\
" be wrapping the code in try/catch and returning data that contains any updated variables after the exception."
result = PyforaObjectConverter.PyforaObjectConverter()\
.unwrapPyforaDictToDictOfAssignedVars(self.valueIVC)
assert isinstance(result, dict)
return result
@ComputedGraph.ExposedProperty()
def pyforaDictToAssignedVarsToComputedValues(self):
if self.isException:
return self.jsonValueRepresentation
stringDictToIVC = self.pyforaDictToDictOfAssignedVarsToProxyValues
if stringDictToIVC is None:
return None
return {
'isException': False,
'dictOfProxies': {k: PyforaDictionaryElement(baseCV=self, keyname=k) for k in stringDictToIVC}
}
def pyforaTupleToTuple(self):
if self.valueIVC is None:
return None
result = PyforaObjectConverter.PyforaObjectConverter().unwrapPyforaTupleToTuple(self.valueIVC)
assert isinstance(result, tuple)
return result
@ComputedGraph.ExposedProperty()
def pyforaTupleToTupleOfComputedValues(self):
if self.isException:
return self.jsonValueRepresentation
tupleIVC = self.pyforaTupleToTuple
if tupleIVC is None:
return None
return {
'isException': False,
'tupleOfComputedValues': tuple([PyforaTupleElement(baseCV=self, index=ix) for ix in range(len(tupleIVC))])
}
@ComputedGraph.ExposedProperty()
def jsonStatusRepresentation(self):
"""Indicate the current status of the computation.
States:
None - the computation is unfinished
{'status': 'failure', 'message': message} - the computation failed for some unhandleable reason
{'status': 'exception'} - the computation produced an exception
{'status': 'result'} - the computation produced a result
"""
if self.isFailure:
message = None
# Extracts the ErrorState object, defined in ufora/FORA/Core/ErrorState
failure = self.result.asFailure.error
if failure.isHalt():
message = "Computation halted: %s" % failure.asHalt.uuid
elif failure.isIllegalComputationState():
message = str(failure.asIllegalComputationState.m0)
elif failure.isMemoryQuotaExceeded():
memoryQuotaFailure = failure.asMemoryQuotaExceeded
message = "Memory quota exceeded (amount: %s, required: %s)" % \
(memoryQuotaFailure.amount, memoryQuotaFailure.required)
return {'status': 'failure', 'message': message}
if self.valueIVC is None:
return None
if self.isException:
return {'status': 'exception'}
else:
return {'status': 'result'}
@ComputedGraph.ExposedProperty()
def jsonValueRepresentation(self):
return PyforaResultAsJson(computedValue=self, maxBytecount=None).getResultAsJson()
def exceptionValueAsString(self):
if not self.isException:
return None
if self.valueIVC.isTuple():
exception, stacktraceAndVarsInScope = self.valueIVC.getTuple()
return self.unwrapExceptionIVC(exception)
else:
return self.unwrapExceptionIVC(self.valueIVC)
@ComputedGraph.Function
def unwrapExceptionIVC(self, exceptionIVC):
try:
return str(ForaValue.FORAValue(exceptionIVC))
except:
logging.error("calling 'str' on %s failed: %s", exceptionIVC, traceback.format_exc())
return "<unknown exception>"
def exceptionCodeLocationsAsJson(self):
if self.valueIVC.isTuple():
tup = self.valueIVC.getTuple()
if len(tup) != 2:
return None
_, stacktrace = self.valueIVC.getTuple()
codeLocations = stacktrace.getStackTrace()
if codeLocations is None:
return None
def formatCodeLocation(c):
if c is None:
return None
if not c.defPoint.isExternal():
return None
def posToJson(simpleParsePosition):
return {
'line': simpleParsePosition.line,
'col': simpleParsePosition.col
}
return {
'path': list(c.defPoint.asExternal.paths),
'range': {
'start': posToJson(c.range.start),
'stop': posToJson(c.range.stop)
}
}
# return [x for x in [formatCodeLocation(c) for c in codeLocations] if x is not None]
return [x for x in [formatCodeLocation(c) for c in codeLocations if c is not None] if x is not None]
else:
return None
