class PyforaTupleElement(PyforaComputedValue): #the base PyforaComputedValue of which we are one of the tuple members baseCV = ComputedGraph.Key(object) #the index in self.baseCV.pyforaTupleToTuple we are tied to index = ComputedGraph.Key(object) def argIds(self): return () def args(self): return (self.baseCV, ForaNative.makeSymbol("RawGetItemByInt"), ForaNative.ImplValContainer(self.index)) def __str__(self): return "PyforaTupleElement(baseCV=%s,index=%s)" % (self.baseCV, self.index)
class PyforaDictionaryElement(PyforaComputedValue): #the base PyforaComputedValue of which we are one of the dictionary members baseCV = ComputedGraph.Key(object) #the keyname in self.baseCV.pyforaDictToStringDict we are tied to keyname = ComputedGraph.Key(object) def argIds(self): return () def args(self): return (self.baseCV, ForaNative.makeSymbol("RawGetItemByString"), ForaNative.ImplValContainer(self.keyname)) def __str__(self): return "PyforaDictionaryElement(baseCV=%s,keyname=%s)" % (self.baseCV, self.keyname)
class TestCGLocation(ComputedGraph.Location): definition = ComputedGraph.Key(object) aValue = ComputedGraph.Mutable(lambda: None, exposeToProtocol=True) def sharedStateSubspace(self): return CGSS.Node.Keyspace(keyspacePath=("public", "writeable", "TestCGLocation")).subspace aValueInSharedState = CGSS.Property.Property(default=lambda: False, exposeToProtocol=True) bValueInSharedState = CGSS.Property.Property(default=lambda: False, exposeToProtocol=True) cValueInSharedState = CGSS.Property.Property(default=lambda: False, exposeToProtocol=True) @ComputedGraph.ExposedProperty() def depth(self): if isinstance(self.definition, Test): return self.definition.depth + 1 elif isinstance(self.definition, list): res = 0 for x in self.definition: res += x.depth return res else: return 0 @ComputedGraph.ExposedProperty() def testCgLocation(self): return self @ComputedGraph.ExposedFunction() def aFunction(self, jsonArg): self.aValue = jsonArg @ComputedGraph.Function def anUnexposedFunction(self, jsonArg): self.aValue = jsonArg @ComputedGraph.ExposedFunction() def testFunction(self, arg): return arg @ComputedGraph.ExposedFunction(wantsCallback=True) def aFunctionExpectingCallback(self, callback, jsonArg): def aThread(): time.sleep(1) callback(jsonArg) threading.Thread(target=aThread).start()
class ComputedValueVector(ComputedGraph.Location): """A Location representing a loadable piece of data in the Gateway's RAM cache. Incrementing its request count causes it to try to be loaded, and decrementing it gets rid of the dependency. Clients can access the data as a numpy array or pull out individual FORA values. """ vectorImplVal = ComputedGraph.Key(object, default = None, validator = validateIsVectorIVC) @ComputedGraph.Function def __getitem__(self, index): """if index is an integer, attempt to index into the vector. If index is a slice, produce a computed value with the sliced vector """ if self.vectorImplVal is None: raise IndexError() if isinstance(index, slice): return ComputedValue( args = ( self.vectorImplVal, ForaNative.makeSymbol("GetItemDeepcopied"), ForaNative.ImplValContainer(index.start), ForaNative.ImplValContainer(index.stop), ForaNative.ImplValContainer(index.step) ) ) if index < 0 or index >= self.elementCount: raise IndexError() return self.entireSlice.extractVectorItemAsIVC(index) @ComputedGraph.ExposedProperty() def elementCount(self): if self.vectorImplVal is None: return 0 return self.vectorImplVal.getVectorSize() def entireSlice(self): return self.getMappableSlice(0, self.elementCount) def slicesByPage(self): if self.vectorImplVal is None: return [] return [self.getMappableSlice(low,high) for low,high in self.vectorImplVal.getVectorPageSliceRanges( ComputedValueGateway.getGateway().vdm )] @ComputedGraph.Function def getMappableSlice(self, lowIndex, highIndex): return ComputedValueVectorSlice( computedValueVector = self, lowIndex = lowIndex, highIndex = highIndex ) @ComputedGraph.ExposedFunction(expandArgs=True) def getSlice(self, lowIndex, highIndex): return self.getMappableSlice(lowIndex, highIndex) def __str__(self): return "%s" % (self.vectorImplVal)
class ComputedValue(ComputedGraph.Location): """Represents the description of a single Fora computation.""" # A tuple that represents the axiom-like Fora expression to compute. # e.g. (implValForFunction, `Call, 1, 2) args = ComputedGraph.Key(object, default = None, validator = validateComputedValueArgs) # Result is an Interpreter::ComputationResult (defined in # ufora/FORA/Core/ComputationResult.hppml) # which may consist of an Exception, Result, or Failure. result = ComputedGraph.Mutable(lambda: None) computationStatistics = ComputedGraph.Mutable(lambda: None) def __hash__(self): return hash(self.hash) def __cmp__(self, other): self.depth return TypeAwareComparison.typecmp(self, other, lambda self, other : cmp(self.hash, other.hash)) def depth(self): """compute the depth of the ComputedValue tree""" tr = 0 if self.args is None: return 0 for a in self.args: try: tr = max(a.depth + 1, tr) except AttributeError: pass assert tr < 50, self return tr @ComputedGraph.ExposedProperty() def asVector(self): if not self.isFinished: return None if self.isException: return None if not self.valueIVC.isVector(): return None return ComputedValueVectorFromComputedValue(computedValue = self) def computationDefinitionTerm_(self): return CumulusNative.ComputationDefinitionTerm.Subcomputation( self.cumulusComputationDefinition.asRoot.terms ) def cumulusComputationDefinition(self): terms = [] assert self.args is not None, self.__location_class__ for a in self.args: if isinstance(a, (long, int, str, bool)): terms.append(CumulusNative.ComputationDefinitionTerm.Value(ImplValContainer_(a), None)) elif isinstance(a, ImplValContainer_): terms.append(CumulusNative.ComputationDefinitionTerm.Value(a, None)) else: terms.append(a.computationDefinitionTerm_) return CumulusNative.ComputationDefinition.Root( CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(terms) ) @ComputedGraph.ExposedProperty() def submittedComputationId(self): computationId = ComputedValueGateway.getGateway().submittedComputationId(self.cumulusComputationDefinition) if computationId is None: return return computationId.toSimple() @ComputedGraph.ExposedFunction() def cancel(self, *args): ComputedValueGateway.getGateway().cancelComputation(self, self.cumulusComputationDefinition) @ComputedGraph.ExposedFunction() def requestComputationCheckpoint(self, *args): ComputedValueGateway.getGateway().requestComputationCheckpoint(self, self.cumulusComputationDefinition) @ComputedGraph.ExposedFunction() def increaseRequestCount(self, *args): ComputedValueGateway.getGateway().increaseRequestCount(self, self.cumulusComputationDefinition) @ComputedGraph.ExposedFunction() def decreaseRequestCount(self, *args): ComputedValueGateway.getGateway().decreaseRequestCount(self, self.cumulusComputationDefinition) def totalVectorBytesReferenced(self): if self.checkpointStatus is None: return 0 stats = self.checkpointStatus.statistics return ComputedValueGateway.getGateway().bytecountForBigvecs( self.checkpointStatus.bigvecsReferenced ) def totalBytesOfMemoryReferenced(self): if self.checkpointStatus is None: return 0 stats = self.checkpointStatus.statistics return stats.totalBytesInMemory def isCompletelyCheckpointed(self): if self.checkpointStatus is None: return False stats = self.checkpointStatus.statistics totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter return self.totalComputeSecondsAtLastCheckpoint + 1.0 > totalSeconds @ComputedGraph.ExposedProperty() def unfinishedDependentCodeLocationsAsJson(self): res = [] for cv in self.rootComputedValueDependencies: if not cv.isFinished: #each computed value we depend on might be a visualize. If so, we want #the actual script computation (if available) cv = cv.unwrapVisualizable cv = cv.unwrapMember #only some ComputedValue objects have this property loc = cv.codeLocationDefinitionAsJson if loc is not None: res.append(loc) else: res.append(str(ComputedGraph.getLocationTypeFromLocation(cv))) return tuple(res) @ComputedGraph.ExposedProperty() def stats(self): if self.checkpointStatus is None: return {} stats = self.checkpointStatus.statistics result = { "status": { "title" : "Computation Status", "value" : "Finished" if self.isFinished else "Unfinished" + ((" (%s cpus)" % self.totalWorkerCount) if self.totalWorkerCount > 0 else ""), "units" : "" }, "timeSpentInCompiler": { "title" : "Time in compiled code (across all cores)", "value" : stats.timeSpentInCompiler, "units" : "sec" }, "timeSpentInInterpreter": { "title" : "Time in interpreted code (across all cores)", "value" : stats.timeSpentInInterpreter, "units" : "sec" }, "totalSplitCount": { "title" : "Total split count", "value" : stats.totalSplitCount, "units" : "" }, "totalBytesReferenced": { "title" : "Total bytes referenced (calculations)", "value" : self.totalBytesOfMemoryReferenced, "units" : "bytes" }, "totalBytesReferencedJustPaged": { "title" : "Total bytes referenced (vectors)", "value" : self.totalVectorBytesReferenced, "units" : "bytes" } } result["isCheckpointing"] = { "title" : "Is Checkpointing", "value" : self.isCheckpointing, "units" : "" } result["isLoadingFromCheckpoint"] = { "title" : "Is loading from checkpoint", "value" : self.isLoadingFromCheckpoint, "units" : "" } if self.totalBytesReferencedAtLastCheckpoint > 0: result["totalBytesReferencedAtLastCheckpoint"] = { 'title': "Size of last checkpoint", 'value': self.totalBytesReferencedAtLastCheckpoint, 'units': 'bytes' } secondsAtCheckpoint = self.totalComputeSecondsAtLastCheckpoint if secondsAtCheckpoint == 0.0: result["checkpointStatus"] = { "title" : "Checkpoint Status", "value" : "not checkpointed", "units" : "" } else: totalSeconds = stats.timeSpentInCompiler + stats.timeSpentInInterpreter if secondsAtCheckpoint + 1.0 >= totalSeconds: result["checkpointStatus"] = { "title" : "Checkpoint Status", "value" : "Checkpointed", "units" : "" } else: result["checkpointStatus"] = { "title" : "Uncheckpointed compute seconds", "value" : totalSeconds - secondsAtCheckpoint, "units" : "sec" } return result def isFailure(self): if not self.result: return None return self.result.isFailure() def failure(self): if not self.result: return None return self.result.asFailure.error.toString() @ComputedGraph.ExposedProperty() def isException(self): if not self.result: return None return self.result.isException() def valueIVC(self): if not self.result: return None if self.result.isException(): return self.result.asException.exception if self.result.isResult(): return self.result.asResult.result return None @ComputedGraph.ExposedProperty() def isFinished(self): return self.valueIVC is not None or self.isFailure def isEmpty(self): if not self.isFinished: return True if self.isException: return False if self.isFailure: return False if self.valueIVC is None: return True if self.isVector: return self.valueIVC.getVectorSize() == 0 if self.valueIVC.isNothing(): return True if self.valueIVC.isString(): return len(self.valueIVC.pyval) == 0 if self.valueIVC.isTuple(): return len(self.valueIVC) == 0 return False workerCount = ComputedGraph.Mutable(object, lambda: 0) workerCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0) cacheloadCount = ComputedGraph.Mutable(object, lambda: 0) cacheloadCountForDependentComputations = ComputedGraph.Mutable(object, lambda: 0) checkpointStatus = ComputedGraph.Mutable(object, lambda: None) totalComputeSecondsAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0.0) isCheckpointing = ComputedGraph.Mutable(object, lambda: False) isLoadingFromCheckpoint = ComputedGraph.Mutable(object, lambda: False) rootComputedValueDependencies = ComputedGraph.Mutable(object, lambda: ()) totalBytesReferencedAtLastCheckpoint = ComputedGraph.Mutable(object, lambda: 0) @ComputedGraph.ExposedProperty() def totalWorkerCount(self): return self.workerCount + self.workerCountForDependentComputations def codeLocationDefinitionAsJson(self): return None @ComputedGraph.ExposedProperty() def totalCacheloadCount(self): return self.cacheloadCount + self.cacheloadCountForDependentComputations def __str__(self): if self.args is None: return "ComputedValue(None)" return "ComputedValue" + str(tuple(self.args)) def hash(self): h0 = self.hashValue_(self.args[0]) for arg in self.args[1:]: h0 = h0 + self.hashValue_(arg) return h0 @ComputedGraph.Function def hashValue_(self, value): if hasattr(value, 'hash'): return value.hash logging.debug("Using python hash on type '%s'. %s", type(value), str(value)) hashValue = ctypes.c_uint32(hash(value)).value return Hash.Hash(hashValue) def isVector(self): if self.isFailure: return False if self.valueIVC is None: return False return self.valueIVC.isVector() @ComputedGraph.ExposedFunction(expandArgs=True) def writeToS3(self, bucketname, keyname): """Trigger a task writing this dataset to amazon S3. Returns a WriteToS3Task object. """ if not isinstance(bucketname, str): raise Exceptions.SubscribableWebObjectsException("Expected bucketname to be a string") if not isinstance(keyname, str): raise Exceptions.SubscribableWebObjectsException("Expected keyname to be a string") if self.valueIVC is None: return None task = WriteToS3Task.WriteToS3Task(computedValue=self, bucketname=bucketname, keyname=keyname) task.trigger() return task
class ComputedValueVectorFromComputedValue(ComputedValueVector): computedValue = ComputedGraph.Key(object) def vectorImplVal(self): return self.computedValue.valueIVC
class Keyspace(ComputedGraph.Location): keyspacePath = ComputedGraph.Key(object, validator=isTuple) isLoaded_ = ComputedGraph.Mutable(object, lambda: False) toCallOnLoad_ = ComputedGraph.Mutable(object, lambda: ()) isSubscribed_ = ComputedGraph.Mutable(object, lambda: False) @ComputedGraph.Function def onLoad(self, toCallOnLoad): if self.isLoaded_: toCallOnLoad() else: if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent( ) is not None: self.waitLoaded() toCallOnLoad() return else: self.toCallOnLoad_ = self.toCallOnLoad_ + (toCallOnLoad, ) self.ensureSubscribed() def keyspaceName(self): return keyspacePathToKeyspaceName(self.keyspacePath) @ComputedGraph.Function def __str__(self): return "Keyspace(%s)" % (self.keyspacePath, ) @ComputedGraph.Function def ensureSubscribed(self): assert SharedStateSynchronizer.isConnected() if not self.isSubscribed_: logging.info("ComputedGraphSharedState subscribing to: %s", self) SharedStateSynchronizer.getSynchronizer().addKeyspaceListener( self.keyspaceName, KeyspaceUpdater(self), NativeJson.Json(())) self.isSubscribed_ = True @ComputedGraph.Function def markLoaded(self): if self.isLoaded_: return logging.info("ComputedGraphSharedState marking loaded: %s", self) self.isLoaded_ = True self.ensureSubscribed() for toCallOnLoad in self.toCallOnLoad_: toCallOnLoad() self.toCallOnLoad_ = () @ComputedGraph.Function def waitLoaded(self): if not self.isLoaded_: self.ensureSubscribed() SharedStateSynchronizer.getSynchronizer().waitForKeyspaceAndPrefix( self.keyspaceName, NativeJson.Json(())) def loaded(self): if (not self.isLoaded_ and SynchronousPropertyAccess. SynchronousPropertyAccess.getCurrent() is not None): self.ensureSubscribed() self.waitLoaded() return True else: self.ensureSubscribed() return self.isLoaded_ def subspace(self): return Subspace(keyspace=self, keyPath=()) @ComputedGraph.Function def subKeyspace(self, subspaceName): return Keyspace(keyspacePath=self.keyspacePath + (subspaceName, )) @ComputedGraph.Function def assertLoaded(self): if not self.loaded: raise NotLoadedException(self)
class Subspace(ComputedGraph.Location): keyspace = object keyPath = ComputedGraph.Key(object, validator=isTuple) value_ = ComputedGraph.Mutable(object, lambda: None) def keyName(self): return NativeJson.Json(('CGSS', JsonPickle.toJson(self.keyPath))) def loaded(self): return self.keyspace.loaded @ComputedGraph.Function def subKeyspace(self, subKeyspaceName): return Subspace(keyspace=self.keyspace.subKeyspace(subKeyspaceName), keyPath=self.keyPath) @ComputedGraph.Function def subspace(self, subspace): return Subspace(keyspace=self.keyspace, keyPath=self.keyPath + (subspace, )) @ComputedGraph.Function def __str__(self): return "Subspace(%s,%s)" % (self.keyspace, self.keyPath) @ComputedGraph.Function def setValueSlot(self, newValue): logging.info("Setting %s %s", str(self), newValue) if not self.keyspace.loaded: if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent( ) is not None: self.keyspace.waitLoaded() else: logging.info("raising NotLoadedException for %s", self) raise NotLoadedException(self.keyspace) if newValue is None: SharedStateSynchronizer.getSynchronizer().writeValue( self.keyspace.keyspaceName, self.keyName, None) else: assert isinstance(newValue, tuple) assert len(newValue) == 1 try: SharedStateSynchronizer.getSynchronizer().writeValue( self.keyspace.keyspaceName, self.keyName, JsonPickle.toJson(newValue[0])) except Exception as e: logging.error("%s", traceback.format_exc()) raise self.value_ = newValue @ComputedGraph.WithSetter(setValueSlot) def value(self): if not self.keyspace.loaded: if SynchronousPropertyAccess.SynchronousPropertyAccess.getCurrent( ) is not None: self.keyspace.waitLoaded() else: logging.info("raising NotLoadedException for %s", self) raise NotLoadedException(self.keyspace) return self.value_
class PyforaComputedValue(ComputedValue.ComputedValue): argIds = ComputedGraph.Key(object, default=None, validator=validateObjectIds) def args(self): converter = PyforaObjectConverter.PyforaObjectConverter() def unwrapArg(argId): if isinstance(argId, int): return converter.getIvcFromObjectId(argId) else: return argId implVals = tuple(unwrapArg(arg) for arg in self.argIds) return implVals[:1] + (ForaValue.FORAValue.symbol_Call.implVal_,) + implVals[1:] def __str__(self): return "PyforaComputedValue" + str(tuple(self.argIds)) def pyforaDictToDictOfAssignedVarsToProxyValues(self): if self.valueIVC is None: return None assert not self.isException, "We should not allow exceptions to be thrown here. Instead we should " +\ " be wrapping the code in try/catch and returning data that contains any updated variables after the exception." result = PyforaObjectConverter.PyforaObjectConverter()\ .unwrapPyforaDictToDictOfAssignedVars(self.valueIVC) assert isinstance(result, dict) return result @ComputedGraph.ExposedProperty() def pyforaDictToAssignedVarsToComputedValues(self): if self.isException: return self.jsonValueRepresentation stringDictToIVC = self.pyforaDictToDictOfAssignedVarsToProxyValues if stringDictToIVC is None: return None return { 'isException': False, 'dictOfProxies': {k: PyforaDictionaryElement(baseCV=self, keyname=k) for k in stringDictToIVC} } def pyforaTupleToTuple(self): if self.valueIVC is None: return None result = PyforaObjectConverter.PyforaObjectConverter().unwrapPyforaTupleToTuple(self.valueIVC) assert isinstance(result, tuple) return result @ComputedGraph.ExposedProperty() def pyforaTupleToTupleOfComputedValues(self): if self.isException: return self.jsonValueRepresentation tupleIVC = self.pyforaTupleToTuple if tupleIVC is None: return None return { 'isException': False, 'tupleOfComputedValues': tuple([PyforaTupleElement(baseCV=self, index=ix) for ix in range(len(tupleIVC))]) } @ComputedGraph.ExposedProperty() def jsonStatusRepresentation(self): """Indicate the current status of the computation. States: None - the computation is unfinished {'status': 'failure', 'message': message} - the computation failed for some unhandleable reason {'status': 'exception'} - the computation produced an exception {'status': 'result'} - the computation produced a result """ if self.isFailure: message = None # Extracts the ErrorState object, defined in ufora/FORA/Core/ErrorState failure = self.result.asFailure.error if failure.isHalt(): message = "Computation halted: %s" % failure.asHalt.uuid elif failure.isIllegalComputationState(): message = str(failure.asIllegalComputationState.m0) elif failure.isMemoryQuotaExceeded(): memoryQuotaFailure = failure.asMemoryQuotaExceeded message = "Memory quota exceeded (amount: %s, required: %s)" % \ (memoryQuotaFailure.amount, memoryQuotaFailure.required) return {'status': 'failure', 'message': message} if self.valueIVC is None: return None if self.isException: return {'status': 'exception'} else: return {'status': 'result'} @ComputedGraph.ExposedProperty() def jsonValueRepresentation(self): return PyforaResultAsJson(computedValue=self, maxBytecount=None).getResultAsJson() def exceptionValueAsString(self): if not self.isException: return None if self.valueIVC.isTuple(): exception, stacktraceAndVarsInScope = self.valueIVC.getTuple() return self.unwrapExceptionIVC(exception) else: return self.unwrapExceptionIVC(self.valueIVC) @ComputedGraph.Function def unwrapExceptionIVC(self, exceptionIVC): try: return str(ForaValue.FORAValue(exceptionIVC)) except: logging.error("calling 'str' on %s failed: %s", exceptionIVC, traceback.format_exc()) return "<unknown exception>" def exceptionCodeLocationsAsJson(self): if self.valueIVC.isTuple(): tup = self.valueIVC.getTuple() if len(tup) != 2: return None _, stacktrace = self.valueIVC.getTuple() codeLocations = stacktrace.getStackTrace() if codeLocations is None: return None def formatCodeLocation(c): if c is None: return None if not c.defPoint.isExternal(): return None def posToJson(simpleParsePosition): return { 'line': simpleParsePosition.line, 'col': simpleParsePosition.col } return { 'path': list(c.defPoint.asExternal.paths), 'range': { 'start': posToJson(c.range.start), 'stop': posToJson(c.range.stop) } } # return [x for x in [formatCodeLocation(c) for c in codeLocations] if x is not None] return [x for x in [formatCodeLocation(c) for c in codeLocations if c is not None] if x is not None] else: return None