def test_simple_serialization_and_deserialization(self): typeDescription = TypeDescription.Primitive(23.4) self.assertDeserializationSerializationSucceeded(typeDescription) typeDescription = TypeDescription.Tuple([1, 2, 3, 4, 5]) self.assertDeserializationSerializationSucceeded(typeDescription) typeDescription = TypeDescription.ClassDefinition("source text here", ['test']) self.assertDeserializationSerializationSucceeded(typeDescription)
def mapPythonInstanceToPyforaInstance(self, numpyArray): flattened = numpyArray.flatten() dtypeAsPrim = TypeDescription.dtypeToPrimitive(numpyArray.dtype) if dtypeAsPrim is not None: return PurePythonNumpyArray( numpyArray.shape, TypeDescription.PackedHomogenousData(dtypeAsPrim, flattened.tostring())) else: return PurePythonNumpyArray(numpyArray.shape, flattened.tolist())
def test_numpy_dtype_conversion(self): for array in [ numpy.array([1.0, 2.0]), numpy.zeros(2, dtype=[('f0', '<f8'), ('f1', '<f8')]), numpy.array([1]), numpy.array([False]) ]: dt = array.dtype dt2 = TypeDescription.primitiveToDtype( TypeDescription.dtypeToPrimitive(array.dtype)) self.assertEqual(str(dt.descr), str(dt2.descr))
def convert(self, objectId, objectRegistry, callback): dependencyGraph = objectRegistry.computeDependencyGraph(objectId) objectIdToObjectDefinition = { objId: TypeDescription.serialize(objectRegistry.getDefinition(objId)) for objId in dependencyGraph.iterkeys() } def onSuccess(message): if 'isException' not in message: callback(objectId) else: callback( Exceptions.PythonToForaConversionError( str(message['message']), message['trace'])) self.remoteConverter.convert( { 'objectId': objectId, 'objectIdToObjectDefinition': objectIdToObjectDefinition }, { 'onSuccess': onSuccess, 'onFailure': lambda err: callback( Exceptions.PythonToForaConversionError(err)) })
def defineClassInstance(self, objectId, classId, classMemberNameToClassMemberId): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.ClassInstanceDescription( classId=classId, classMemberNameToClassMemberId=classMemberNameToClassMemberId )
def defineWithBlock(self, objectId, freeVariableMemberAccessChainsToId, sourceFileId, lineNumber): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.WithBlockDescription( freeVariableMemberAccessChainsToId, sourceFileId, lineNumber )
def defineFunction(self, objectId, sourceFileId, lineNumber, scopeIds): """ scopeIds: a dict freeVariableMemberAccessChain -> id """ freeVariableMemberAccessChainsToId = \ self._processFreeVariableMemberAccessChainResolution(scopeIds) self.objectIdToObjectDefinition[objectId] = TypeDescription.FunctionDefinition( sourceFileId=sourceFileId, lineNumber=lineNumber, freeVariableMemberAccessChainsToId=freeVariableMemberAccessChainsToId )
def defineClass(self, objectId, sourceFileId, lineNumber, scopeIds, baseClassIds): """ scopeIds: a dict freeVariableMemberAccessChain -> id baseClassIds: a list of ids representing (immediate) base classes """ freeVariableMemberAccessChainsToId = \ self._processFreeVariableMemberAccessChainResolution( scopeIds ) self.objectIdToObjectDefinition[objectId] = \ TypeDescription.ClassDefinition( sourceFileId=sourceFileId, lineNumber=lineNumber, freeVariableMemberAccessChainsToId=freeVariableMemberAccessChainsToId, baseClassIds=baseClassIds )
def _computeDependentIds(self, objectId): objectDefinition = self.objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or \ isinstance(objectDefinition, (TypeDescription.File, TypeDescription.RemotePythonObject, TypeDescription.NamedSingleton, list, TypeDescription.Unconvertible, TypeDescription.PackedHomogenousData, TypeDescription.UnresolvedVarWithPosition )): return [] elif isinstance(objectDefinition, (TypeDescription.BuiltinExceptionInstance)): return [objectDefinition.argsId] elif isinstance(objectDefinition, (TypeDescription.List, TypeDescription.Tuple)): return objectDefinition.memberIds elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)): tr = objectDefinition.freeVariableMemberAccessChainsToId.values() tr.append(objectDefinition.sourceFileId) return tr elif isinstance(objectDefinition, TypeDescription.InstanceMethod): return [objectDefinition.instanceId] elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription): tr = [objectDefinition.classId] tr.extend(self._computeDependentIds(objectDefinition.classId)) classMemberIds = \ objectDefinition.classMemberNameToClassMemberId.values() tr.extend(classMemberIds) return tr elif isinstance(objectDefinition, TypeDescription.Dict): return objectDefinition.keyIds + objectDefinition.valueIds elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): tr = objectDefinition.freeVariableMemberAccessChainsToId.values() tr.append(objectDefinition.sourceFileId) return tr else: assert False, "don't know what to do with %s" % type( objectDefinition)
def _computeDependentIds(self, objectId): objectDefinition = self.objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or \ isinstance(objectDefinition, (TypeDescription.File, TypeDescription.RemotePythonObject, TypeDescription.NamedSingleton, list, TypeDescription.Unconvertible)): return [] elif isinstance(objectDefinition, (TypeDescription.BuiltinExceptionInstance)): return [objectDefinition.argsId] elif isinstance(objectDefinition, (TypeDescription.List, TypeDescription.Tuple)): return objectDefinition.memberIds elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)): tr = objectDefinition.freeVariableMemberAccessChainsToId.values() tr.append(objectDefinition.sourceFileId) return tr elif isinstance(objectDefinition, TypeDescription.InstanceMethod): return [objectDefinition.instanceId] elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription): tr = [objectDefinition.classId] tr.extend( self._computeDependentIds( objectDefinition.classId ) ) classMemberIds = \ objectDefinition.classMemberNameToClassMemberId.values() tr.extend(classMemberIds) return tr elif isinstance(objectDefinition, TypeDescription.Dict): return objectDefinition.keyIds + objectDefinition.valueIds elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): tr = objectDefinition.freeVariableMemberAccessChainsToId.values() tr.append(objectDefinition.sourceFileId) return tr else: assert False, "don't know what to do with %s" % type(objectDefinition)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition): objectDefinition = objectIdToObjectDefinition[objectId] logging.info("ObjectDefinition: %s", objectDefinition) if TypeDescription.isPrimitive(objectDefinition) or isinstance( objectDefinition, list): return self.convertPrimitive(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): return self.convertRemotePythonObject(objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): return self.convertNamedSingleton(objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): return self.convertBuiltinExceptionInstance(objectDefinition) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition, TypeDescription.ClassInstanceDescription, TypeDescription.InstanceMethod)): return (self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition)) elif isinstance(objectDefinition, TypeDescription.List): return (self.convertList(objectId, dependencyGraph, objectIdToObjectDefinition)) elif isinstance(objectDefinition, TypeDescription.Tuple): return (self.convertTuple(objectId, dependencyGraph, objectIdToObjectDefinition)) elif isinstance(objectDefinition, TypeDescription.Dict): return (self.convertDict(objectId, objectDefinition, dependencyGraph, objectIdToObjectDefinition)) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): return (self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition)) else: raise pyfora.PythonToForaConversionError( "don't know how to convert %s of type %s" % (objectDefinition, type(objectDefinition)))
def convert(self, objectId, objectRegistry, callback): dependencyGraph = objectRegistry.computeDependencyGraph(objectId) objectIdToObjectDefinition = { objId: TypeDescription.serialize(objectRegistry.getDefinition(objId)) for objId in dependencyGraph.iterkeys() } def onSuccess(message): if 'isException' not in message: callback(objectId) else: callback(Exceptions.PythonToForaConversionError(str(message['message']), message['trace'])) self.remoteConverter.convert( { 'objectId': objectId, 'objectIdToObjectDefinition': objectIdToObjectDefinition }, { 'onSuccess': onSuccess, 'onFailure': lambda err: callback(Exceptions.PythonToForaConversionError(err)) })
def convert(self, objectId, objectIdToObjectDefinition): import pyfora.TypeDescription as TypeDescription import pyfora.Exceptions as PyforaExceptions result = [None] def onConverted(r): result[0] = r t0 = time.time() objectRegistry_[0].objectIdToObjectDefinition.update({ int(k): TypeDescription.deserialize(v) for k, v in objectIdToObjectDefinition.iteritems() }) logging.info("Updated object registry in %s seconds.", time.time() - t0) t0 = time.time() try: converter_[0].convert(objectId, objectRegistry_[0], onConverted) except Exception as e: logging.error("Converter raised an exception: %s", traceback.format_exc()) raise Exceptions.InternalError("Unable to convert objectId %s" % objectId) logging.info("Converted to fora in %s seconds", time.time() - t0) assert result[0] is not None if isinstance(result[0], PyforaExceptions.PythonToForaConversionError): return {'isException': True, 'message': result[0].message, 'trace': result[0].trace} if isinstance(result[0], Exception): raise Exceptions.SubscribableWebObjectsException(result[0].message) objectIdToIvc_[objectId] = result[0] return {'objectId': objectId}
def defineBuiltinExceptionInstance(self, objectId, typename, argsId): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.BuiltinExceptionInstance(typename, argsId)
def defineRemotePythonObject(self, objectId, computedValueArg): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.RemotePythonObject(computedValueArg)
def defineDict(self, objectId, keyIds, valueIds): self.objectIdToObjectDefinition[objectId] = TypeDescription.Dict(keyIds=keyIds, valueIds=valueIds)
def defineFile(self, objectId, text, path): self.objectIdToObjectDefinition[objectId] = TypeDescription.File(path, text)
def deserialize(self, objectDefinition): self.assertTrue(hasattr(objectDefinition, 'typeName')) return TypeDescription.deserialize(objectDefinition)
def deserializeFromStream(stream, objectVisitor, convertJsonToObject): while True: objectId = stream.readInt64() #this is the termination condition if objectId == -1: return code = stream.readByte() def readSimplePrimitive(): code = stream.readByte() if code == BinaryObjectRegistry.CODE_NONE: return None if code == BinaryObjectRegistry.CODE_INT: return stream.readInt64() if code == BinaryObjectRegistry.CODE_STR: return stream.readString() if code == BinaryObjectRegistry.CODE_TUPLE: ct = stream.readInt32() return tuple([readSimplePrimitive() for _ in xrange(ct)]) if code == BinaryObjectRegistry.CODE_LIST: ct = stream.readInt32() return [readSimplePrimitive() for _ in xrange(ct)] if code == BinaryObjectRegistry.CODE_DICT: ct = stream.readInt32() return dict([(readSimplePrimitive(),readSimplePrimitive()) for _ in xrange(ct)]) else: assert False, "unknown code: " + str(code) def readInt64s(): return [stream.readInt64() for _ in xrange(stream.readInt64())] def readStringTuple(): return tuple([stream.readString() for _ in xrange(stream.readInt32())]) def readDottedScopeIds(): res = {} ct = stream.readInt32() for _ in xrange(ct): path = stream.readString() objId = stream.readInt64() res[path] = objId return res def readPrimitive(code): if code == BinaryObjectRegistry.CODE_NONE: return None elif code == BinaryObjectRegistry.CODE_INT: return stream.readInt64() elif code == BinaryObjectRegistry.CODE_LONG: return long(stream.readString()) elif code == BinaryObjectRegistry.CODE_FLOAT: return stream.readFloat64() elif code == BinaryObjectRegistry.CODE_BOOL: return True if stream.readByte() else False elif code == BinaryObjectRegistry.CODE_STR: return stream.readString() elif code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES: return [readPrimitive(stream.readByte()) for _ in xrange(stream.readInt64())] else: assert False, "unknown code: " + str(code) if (code == BinaryObjectRegistry.CODE_NONE or code == BinaryObjectRegistry.CODE_INT or code == BinaryObjectRegistry.CODE_LONG or code == BinaryObjectRegistry.CODE_FLOAT or code == BinaryObjectRegistry.CODE_BOOL or code == BinaryObjectRegistry.CODE_STR or code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES): objectVisitor.definePrimitive(objectId, readPrimitive(code)) elif code == BinaryObjectRegistry.CODE_TUPLE: objectVisitor.defineTuple(objectId, readInt64s()) elif code == BinaryObjectRegistry.CODE_PACKED_HOMOGENOUS_DATA: dtype = readSimplePrimitive() packedBytes = stream.readString() objectVisitor.definePackedHomogenousData(objectId, TypeDescription.PackedHomogenousData(dtype, packedBytes)) elif code == BinaryObjectRegistry.CODE_LIST: objectVisitor.defineList(objectId, readInt64s()) elif code == BinaryObjectRegistry.CODE_FILE: path = stream.readString() text = stream.readString() objectVisitor.defineFile(objectId, text, path) elif code == BinaryObjectRegistry.CODE_DICT: objectVisitor.defineDict(objectId, readInt64s(), readInt64s()) elif code == BinaryObjectRegistry.CODE_REMOTE_PY_OBJECT: jsonRepresentation = json.loads(stream.readString()) objectVisitor.defineRemotePythonObject(objectId, convertJsonToObject(jsonRepresentation)) elif code == BinaryObjectRegistry.CODE_BUILTIN_EXCEPTION_INSTANCE: objectVisitor.defineBuiltinExceptionInstance(objectId, stream.readString(), stream.readInt64()) elif code == BinaryObjectRegistry.CODE_NAMED_SINGLETON: objectVisitor.defineNamedSingleton(objectId, stream.readString()) elif code == BinaryObjectRegistry.CODE_FUNCTION: objectVisitor.defineFunction(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds()) elif code == BinaryObjectRegistry.CODE_CLASS: objectVisitor.defineClass(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds(), readInt64s()) elif code == BinaryObjectRegistry.CODE_UNCONVERTIBLE: if stream.readByte() != 0: objectVisitor.defineUnconvertible(objectId, readStringTuple()) else: objectVisitor.defineUnconvertible(objectId, None) elif code == BinaryObjectRegistry.CODE_CLASS_INSTANCE: classId = stream.readInt64() classMembers = {} for _ in xrange(stream.readInt32()): memberName = stream.readString() classMembers[memberName] = stream.readInt64() objectVisitor.defineClassInstance(objectId, classId, classMembers) elif code == BinaryObjectRegistry.CODE_INSTANCE_METHOD: objectVisitor.defineInstanceMethod(objectId, stream.readInt64(), stream.readString()) elif code == BinaryObjectRegistry.CODE_WITH_BLOCK: scopes = readDottedScopeIds() objectVisitor.defineWithBlock(objectId, scopes, stream.readInt64(), stream.readInt32()) elif code == BinaryObjectRegistry.CODE_PY_ABORT_EXCEPTION: typename = stream.readString() argsId = stream.readInt64() objectVisitor.definePyAbortException(objectId, typename, argsId) elif code == BinaryObjectRegistry.CODE_STACKTRACE_AS_JSON: stackAsJson = stream.readString() objectVisitor.defineStacktrace(objectId, json.loads(stackAsJson)) elif code == BinaryObjectRegistry.CODE_UNRESOLVED_SYMBOL: varname = stream.readString() lineno = stream.readInt64() col_offset = stream.readInt64() objectVisitor.defineUnresolvedVarWithPosition(objectId, varname, lineno, col_offset) else: assert False, "BinaryStreamDeserializer.deserializeFromStream: unknown code: " + str(code)
def definePrimitive(self, objectId, primitive): self.objectIdToObjectDefinition[objectId] = TypeDescription.Primitive( primitive)
def convertStronglyConnectedComponentWithOneNode( self, dependencyGraph, stronglyConnectedComponent, objectIdToObjectDefinition): objectId = stronglyConnectedComponent[0] objectDefinition = objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or isinstance( objectDefinition, list): self.convertedValues[objectId] = self.convertPrimitive( objectDefinition) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)): self.convertStronglyConnectedComponentWithOneFunctionOrClass( objectId, objectDefinition, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription): self.convertClassInstanceDescription(objectId, objectDefinition) elif isinstance(objectDefinition, TypeDescription.List): self.convertedValues[objectId] = self.convertList( objectId, self._computeRestrictedGraph(objectId, dependencyGraph), objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.Tuple): self.convertedValues[objectId] = self.convertTuple( objectId, self._computeRestrictedGraph(objectId, dependencyGraph), objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.Dict): self.convertedValues[objectId] = self.convertDict( objectId, objectDefinition, self._computeRestrictedGraph(objectId, dependencyGraph), objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.File): self.convertedValues[objectId] = self.convertFile(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): self.convertedValues[objectId] = self.convertRemotePythonObject( objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): self.convertedValues[ objectId] = self.convertBuiltinExceptionInstance( objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): self.convertedValues[objectId] = self.convertNamedSingleton( objectDefinition) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): self.convertedValues[objectId] = self.convertWithBlock( objectId, objectDefinition, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.InstanceMethod): self.convertedValues[objectId] = self.convertInstanceMethod( objectId, objectDefinition, objectIdToObjectDefinition) else: assert False, "haven't gotten to this yet %s" % type( objectDefinition)
def test_nested_type_serialization_and_deserialization(self): dict1 = { "a": 1, "b": 2, "c": 3} dict2 = { "a": 4, "b": 5, "c": 6} typeDescription = TypeDescription.ClassDefinition("source text here", [dict1, dict2]) self.assertDeserializationSerializationSucceeded(typeDescription)
def transformBody(objId, implval): if isUnconvertibleValueTuple(implval): path = implval[0].pyval stream.defineUnconvertible(objId, path) return objId if isPyforaNameErrorTuple(implval): payload = implval[0] varname = payload[0].pyval lineno = payload[1].pyval col_offset = payload[2].pyval stream.defineUnresolvedVarWithPosition( objId, varname, lineno, col_offset) return objId if implval.isString(): value = (implval.pyval,) else: value = self.nativeConverterAdaptor.invertForaConstant(implval) if value is not None: if isinstance(value, tuple): #this is a simple constant stream.definePrimitive(objId, value[0]) return objId else: #this is a vector that represents a string assert isinstance(value, ForaNative.ImplValContainer) if len(value) == 0: stream.definePrimitive(objId, "") return objId assert value.isVectorOfChar(), value contents = vectorContentsExtractor(value) if contents is None: anyNeedLoading[0] = True stream.definePrimitive(objId, "") return objId else: assert 'string' in contents stream.definePrimitive(objId, contents['string']) return objId if self.singletonAndExceptionConverter is not None: value = self.singletonAndExceptionConverter.convertInstanceToSingletonName(implval) if value is not None: stream.defineNamedSingleton(objId, value) return objId value = self.singletonAndExceptionConverter.convertExceptionInstance(implval) if value is not None: stream.defineBuiltinExceptionInstance(objId, value[0], transform(value[1])) return objId value = self.singletonAndExceptionConverter.convertPyAbortExceptionInstance( implval ) if value is not None: stream.definePyAbortException(objId, value[0], transform(value[1])) return objId value = self.nativeConverterAdaptor.invertTuple(implval) if value is not None: tupleIds = tuple([transform(x) for x in value]) stream.defineTuple(objId, tupleIds) return objId value = self.nativeConverterAdaptor.invertDict(implval) if value is not None: stream.defineDict( objId, [transform(k) for k in value.keys()], [transform(v) for v in value.values()] ) return objId listItemsAsVector = self.nativeConverterAdaptor.invertList(implval) if listItemsAsVector is not None: contents = vectorContentsExtractor(listItemsAsVector) if contents is None: stream.defineList(objId, []) anyNeedLoading[0] = True return objId elif 'listContents' in contents: stream.defineList(objId, [transform(x) for x in contents['listContents']]) return objId else: assert 'contentsAsNumpyArray' in contents contentsAsNumpy = contents['contentsAsNumpyArray'] stream.definePackedHomogenousData( objId, TypeDescription.PackedHomogenousData( TypeDescription.dtypeToPrimitive(contentsAsNumpy.dtype), contentsAsNumpy.tostring() ) ) return objId if implval.isStackTrace(): stackTraceAsJsonOrNone = self.getStackTraceAsJsonOrNone(implval) if stackTraceAsJsonOrNone is not None: stream.defineStacktrace(objId, stackTraceAsJsonOrNone) return objId else: assert False, "unknown tuple, but not a stacktrace: %s" % implval if implval.isObject(): objectClass = implval.getObjectClass() if objectClass == self.pyforaBoundMethodClass: nameAsImplval = implval.getObjectLexicalMember("@name")[0] if not nameAsImplval.isSymbol(): raise pyfora.ForaToPythonConversionError( "PyBoundMethod found with name %s of type %s, which should be a symbol but is not." % (nameAsImplval, nameAsImplval.type) ) stream.defineInstanceMethod( objId, transform(implval.getObjectLexicalMember("@self")[0]), nameAsImplval.pyval[1:] ) return objId sourcePathAndLine = self.extractSourcePathAndLine(implval) if sourcePathAndLine is not None: if objectClass is not None: classObjectId = transform(objectClass) members = {} for memberName in objectClass.objectMembers: if memberName is not None: member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: assert memberName == "@m" assert member[0].isTuple() membersTuple = member[0] memberNames = membersTuple.getTupleNames() for i, name in enumerate(memberNames): result = transform(membersTuple[i]) if result is not UnconvertibleToken: members[str(name)] = transform(membersTuple[i]) stream.defineClassInstance(objId, classObjectId, members) return objId else: members = {} lexicalMembers = implval.objectLexicalMembers for memberAndBindingSequence in lexicalMembers.iteritems(): #if the binding sequence is empty, then this binding refers to 'self' if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]: memberName = memberAndBindingSequence[0] member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: if member[0] != Symbol_uninitialized: transformed = transform(member[0]) if transformed is not UnconvertibleToken: members[(str(memberName),)] = transformed sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0]) stream.defineFunction( objId, sourceFileId, sourcePathAndLine[2], members ) return objId elif implval.isClass(): members = {} sourcePathAndLine = self.extractSourcePathAndLine(implval) lexicalMembers = implval.objectLexicalMembers for memberAndBindingSequence in lexicalMembers.iteritems(): #if the binding sequence is empty, then this binding refers to 'self' if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]: memberName = memberAndBindingSequence[0] member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: result = transform(member[0]) if result is not UnconvertibleToken: members[(str(memberName),)] = result om = implval.objectMetadata if 'classMetadata' in om: om = om['classMetadata'] om = om['sourceText'].objectMetadata sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0]) stream.defineClass( objId, sourceFileId, sourcePathAndLine[2], members, #this is a little wrong. When going python->binary, we would have #a list of base classes. When going pyfora->binary, we're holding the #base classes by name as free variables. () ) return objId logging.error("Failed to convert %s of type %s back to python", implval, str(implval.type)) raise pyfora.ForaToPythonConversionError( "Result cannot be translated back to python." )
def convertStronglyConnectedComponentWithOneNode(self, dependencyGraph, stronglyConnectedComponent, objectIdToObjectDefinition): objectId = stronglyConnectedComponent[0] objectDefinition = objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list): self.convertedValues[objectId] = self.convertPrimitive(objectDefinition) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)): if isinstance(objectDefinition, TypeDescription.ClassDefinition): for baseId in objectDefinition.baseClassIds: if baseId not in self.convertedValues: self._convert(baseId, dependencyGraph, objectIdToObjectDefinition) assert baseId in self.convertedValues self.convertStronglyConnectedComponentWithOneFunctionOrClass( objectId, objectDefinition, objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription): self.convertClassInstanceDescription( objectId, objectDefinition ) elif isinstance(objectDefinition, TypeDescription.List): self.convertedValues[objectId] = self.convertList( objectId, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.Tuple): self.convertedValues[objectId] = self.convertTuple( objectId, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.Dict): self.convertedValues[objectId] = self.convertDict( objectId, objectDefinition, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.File): self.convertedValues[objectId] = self.convertFile(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): self.convertedValues[objectId] = self.convertRemotePythonObject(objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): self.convertedValues[objectId] = self.convertBuiltinExceptionInstance(objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): self.convertedValues[objectId] = self.convertNamedSingleton(objectDefinition) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): self.convertWithBlock(objectId, objectDefinition, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.InstanceMethod): self.convertedValues[objectId] = self.convertInstanceMethod(objectId, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.Unconvertible): self.convertUnconvertibleValue(objectId, objectDefinition.module_path) elif isinstance(objectDefinition, TypeDescription.PackedHomogenousData): self.convertedValues[objectId] = self.convertPackedHomogenousDataAsList( objectId, objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.UnresolvedVarWithPosition): self.convertedValues[objectId] = self.convertUnresolvedVarWithPosition( objectId, objectDefinition ) else: assert False, "haven't gotten to this yet %s" % type(objectDefinition)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition): objectDefinition = objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list): return self.convertPrimitive(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): return self.convertRemotePythonObject(objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): return self.convertNamedSingleton(objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): return self.convertBuiltinExceptionInstance( objectDefinition ) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition, TypeDescription.ClassInstanceDescription, TypeDescription.InstanceMethod) ): return ( self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.List): return ( self.convertList( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.PackedHomogenousData): return ( self.convertPackedHomogenousDataAsList( objectId, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Tuple): return ( self.convertTuple( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Dict): return ( self.convertDict( objectId, objectDefinition, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): return ( self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Unconvertible): return self.convertUnconvertibleValue(objectId, objectDefinition.module_path) elif isinstance(objectDefinition, TypeDescription.UnresolvedVarWithPosition): return self.convertUnresolvedVarWithPosition(objectId, objectDefinition) else: raise pyfora.PythonToForaConversionError( "don't know how to convert %s of type %s" % ( objectDefinition, type(objectDefinition) ) )
def defineNamedSingleton(self, objectId, singletonName): self.objectIdToObjectDefinition[objectId] = TypeDescription.NamedSingleton(singletonName)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition): objectDefinition = objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list): return self.convertPrimitive(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): return self.convertRemotePythonObject(objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): return self.convertNamedSingleton(objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): return self.convertBuiltinExceptionInstance( objectDefinition ) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition, TypeDescription.ClassInstanceDescription, TypeDescription.InstanceMethod) ): return ( self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.List): return ( self.convertList( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Tuple): return ( self.convertTuple( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Dict): return ( self.convertDict( objectId, objectDefinition, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): return ( self.convertObjectWithDependencies( objectId, dependencyGraph, objectIdToObjectDefinition ) ) elif isinstance(objectDefinition, TypeDescription.Unconvertible): return self.convertUnconvertibleValue(objectId) else: raise pyfora.PythonToForaConversionError( "don't know how to convert %s of type %s" % ( objectDefinition, type(objectDefinition) ) )
def defineTuple(self, objectId, memberIds): self.objectIdToObjectDefinition[objectId] = TypeDescription.Tuple(memberIds)
def defineUnconvertible(self, objectId): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.Unconvertible()
def defineList(self, objectId, memberIds): self.objectIdToObjectDefinition[objectId] = TypeDescription.List(memberIds)
def defineInstanceMethod(self, objectId, instanceId, methodName): self.objectIdToObjectDefinition[objectId] = \ TypeDescription.InstanceMethod( instanceId=instanceId, methodName=methodName )
def assertDeserializationSerializationSucceeded(self, typeDescription): jsonString = json.dumps(typeDescription) jobject = json.loads(jsonString) deserializationResult = TypeDescription.fromList(jobject) self.assertNamedTuplesAreTheSame(deserializationResult, typeDescription)
def definePyAbortException(self, objectId, typename, argsId): self.objectIdToObjectDefinition[ objectId] = TypeDescription.PyAbortException(typename, argsId)
def convertStronglyConnectedComponentWithOneNode(self, dependencyGraph, stronglyConnectedComponent, objectIdToObjectDefinition): objectId = stronglyConnectedComponent[0] objectDefinition = objectIdToObjectDefinition[objectId] if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list): self.convertedValues[objectId] = self.convertPrimitive(objectDefinition) elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)): if isinstance(objectDefinition, TypeDescription.ClassDefinition): for baseId in objectDefinition.baseClassIds: if baseId not in self.convertedValues: self._convert(baseId, dependencyGraph, objectIdToObjectDefinition) assert baseId in self.convertedValues self.convertStronglyConnectedComponentWithOneFunctionOrClass( objectId, objectDefinition, objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription): self.convertClassInstanceDescription( objectId, objectDefinition ) elif isinstance(objectDefinition, TypeDescription.List): self.convertedValues[objectId] = self.convertList( objectId, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.Tuple): self.convertedValues[objectId] = self.convertTuple( objectId, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.Dict): self.convertedValues[objectId] = self.convertDict( objectId, objectDefinition, self._computeRestrictedGraph( objectId, dependencyGraph ), objectIdToObjectDefinition ) elif isinstance(objectDefinition, TypeDescription.File): self.convertedValues[objectId] = self.convertFile(objectDefinition) elif isinstance(objectDefinition, TypeDescription.RemotePythonObject): self.convertedValues[objectId] = self.convertRemotePythonObject(objectDefinition) elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance): self.convertedValues[objectId] = self.convertBuiltinExceptionInstance(objectDefinition) elif isinstance(objectDefinition, TypeDescription.NamedSingleton): self.convertedValues[objectId] = self.convertNamedSingleton(objectDefinition) elif isinstance(objectDefinition, TypeDescription.WithBlockDescription): self.convertWithBlock(objectId, objectDefinition, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.InstanceMethod): self.convertedValues[objectId] = self.convertInstanceMethod(objectId, objectIdToObjectDefinition) elif isinstance(objectDefinition, TypeDescription.Unconvertible): self.convertedValues[objectId] = Symbol_unconvertible else: assert False, "haven't gotten to this yet %s" % type(objectDefinition)