def mapPythonInstanceToPyforaInstance(self, numpyArray): flattened = numpyArray.flatten() dtypeAsPrim = TypeDescription.dtypeToPrimitive(numpyArray.dtype) if dtypeAsPrim is not None: return PurePythonNumpyArray( numpyArray.shape, TypeDescription.PackedHomogenousData(dtypeAsPrim, flattened.tostring())) else: return PurePythonNumpyArray(numpyArray.shape, flattened.tolist())
def transformBody(objId, implval): if isUnconvertibleValueTuple(implval): path = implval[0].pyval stream.defineUnconvertible(objId, path) return objId if isPyforaNameErrorTuple(implval): payload = implval[0] varname = payload[0].pyval lineno = payload[1].pyval col_offset = payload[2].pyval stream.defineUnresolvedVarWithPosition( objId, varname, lineno, col_offset) return objId if implval.isString(): value = (implval.pyval,) else: value = self.nativeConverterAdaptor.invertForaConstant(implval) if value is not None: if isinstance(value, tuple): #this is a simple constant stream.definePrimitive(objId, value[0]) return objId else: #this is a vector that represents a string assert isinstance(value, ForaNative.ImplValContainer) if len(value) == 0: stream.definePrimitive(objId, "") return objId assert value.isVectorOfChar(), value contents = vectorContentsExtractor(value) if contents is None: anyNeedLoading[0] = True stream.definePrimitive(objId, "") return objId else: assert 'string' in contents stream.definePrimitive(objId, contents['string']) return objId if self.singletonAndExceptionConverter is not None: value = self.singletonAndExceptionConverter.convertInstanceToSingletonName(implval) if value is not None: stream.defineNamedSingleton(objId, value) return objId value = self.singletonAndExceptionConverter.convertExceptionInstance(implval) if value is not None: stream.defineBuiltinExceptionInstance(objId, value[0], transform(value[1])) return objId value = self.singletonAndExceptionConverter.convertPyAbortExceptionInstance( implval ) if value is not None: stream.definePyAbortException(objId, value[0], transform(value[1])) return objId value = self.nativeConverterAdaptor.invertTuple(implval) if value is not None: tupleIds = tuple([transform(x) for x in value]) stream.defineTuple(objId, tupleIds) return objId value = self.nativeConverterAdaptor.invertDict(implval) if value is not None: stream.defineDict( objId, [transform(k) for k in value.keys()], [transform(v) for v in value.values()] ) return objId listItemsAsVector = self.nativeConverterAdaptor.invertList(implval) if listItemsAsVector is not None: contents = vectorContentsExtractor(listItemsAsVector) if contents is None: stream.defineList(objId, []) anyNeedLoading[0] = True return objId elif 'listContents' in contents: stream.defineList(objId, [transform(x) for x in contents['listContents']]) return objId else: assert 'contentsAsNumpyArray' in contents contentsAsNumpy = contents['contentsAsNumpyArray'] stream.definePackedHomogenousData( objId, TypeDescription.PackedHomogenousData( TypeDescription.dtypeToPrimitive(contentsAsNumpy.dtype), contentsAsNumpy.tostring() ) ) return objId if implval.isStackTrace(): stackTraceAsJsonOrNone = self.getStackTraceAsJsonOrNone(implval) if stackTraceAsJsonOrNone is not None: stream.defineStacktrace(objId, stackTraceAsJsonOrNone) return objId else: assert False, "unknown tuple, but not a stacktrace: %s" % implval if implval.isObject(): objectClass = implval.getObjectClass() if objectClass == self.pyforaBoundMethodClass: nameAsImplval = implval.getObjectLexicalMember("@name")[0] if not nameAsImplval.isSymbol(): raise pyfora.ForaToPythonConversionError( "PyBoundMethod found with name %s of type %s, which should be a symbol but is not." % (nameAsImplval, nameAsImplval.type) ) stream.defineInstanceMethod( objId, transform(implval.getObjectLexicalMember("@self")[0]), nameAsImplval.pyval[1:] ) return objId sourcePathAndLine = self.extractSourcePathAndLine(implval) if sourcePathAndLine is not None: if objectClass is not None: classObjectId = transform(objectClass) members = {} for memberName in objectClass.objectMembers: if memberName is not None: member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: assert memberName == "@m" assert member[0].isTuple() membersTuple = member[0] memberNames = membersTuple.getTupleNames() for i, name in enumerate(memberNames): result = transform(membersTuple[i]) if result is not UnconvertibleToken: members[str(name)] = transform(membersTuple[i]) stream.defineClassInstance(objId, classObjectId, members) return objId else: members = {} lexicalMembers = implval.objectLexicalMembers for memberAndBindingSequence in lexicalMembers.iteritems(): #if the binding sequence is empty, then this binding refers to 'self' if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]: memberName = memberAndBindingSequence[0] member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: if member[0] != Symbol_uninitialized: transformed = transform(member[0]) if transformed is not UnconvertibleToken: members[(str(memberName),)] = transformed sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0]) stream.defineFunction( objId, sourceFileId, sourcePathAndLine[2], members ) return objId elif implval.isClass(): members = {} sourcePathAndLine = self.extractSourcePathAndLine(implval) lexicalMembers = implval.objectLexicalMembers for memberAndBindingSequence in lexicalMembers.iteritems(): #if the binding sequence is empty, then this binding refers to 'self' if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]: memberName = memberAndBindingSequence[0] member = implval.getObjectLexicalMember(memberName) if member is not None and member[1] is None: result = transform(member[0]) if result is not UnconvertibleToken: members[(str(memberName),)] = result om = implval.objectMetadata if 'classMetadata' in om: om = om['classMetadata'] om = om['sourceText'].objectMetadata sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0]) stream.defineClass( objId, sourceFileId, sourcePathAndLine[2], members, #this is a little wrong. When going python->binary, we would have #a list of base classes. When going pyfora->binary, we're holding the #base classes by name as free variables. () ) return objId logging.error("Failed to convert %s of type %s back to python", implval, str(implval.type)) raise pyfora.ForaToPythonConversionError( "Result cannot be translated back to python." )
def deserializeFromStream(stream, objectVisitor, convertJsonToObject): while True: objectId = stream.readInt64() #this is the termination condition if objectId == -1: return code = stream.readByte() def readSimplePrimitive(): code = stream.readByte() if code == BinaryObjectRegistry.CODE_NONE: return None if code == BinaryObjectRegistry.CODE_INT: return stream.readInt64() if code == BinaryObjectRegistry.CODE_STR: return stream.readString() if code == BinaryObjectRegistry.CODE_TUPLE: ct = stream.readInt32() return tuple([readSimplePrimitive() for _ in xrange(ct)]) if code == BinaryObjectRegistry.CODE_LIST: ct = stream.readInt32() return [readSimplePrimitive() for _ in xrange(ct)] if code == BinaryObjectRegistry.CODE_DICT: ct = stream.readInt32() return dict([(readSimplePrimitive(),readSimplePrimitive()) for _ in xrange(ct)]) else: assert False, "unknown code: " + str(code) def readInt64s(): return [stream.readInt64() for _ in xrange(stream.readInt64())] def readStringTuple(): return tuple([stream.readString() for _ in xrange(stream.readInt32())]) def readDottedScopeIds(): res = {} ct = stream.readInt32() for _ in xrange(ct): path = stream.readString() objId = stream.readInt64() res[path] = objId return res def readPrimitive(code): if code == BinaryObjectRegistry.CODE_NONE: return None elif code == BinaryObjectRegistry.CODE_INT: return stream.readInt64() elif code == BinaryObjectRegistry.CODE_LONG: return long(stream.readString()) elif code == BinaryObjectRegistry.CODE_FLOAT: return stream.readFloat64() elif code == BinaryObjectRegistry.CODE_BOOL: return True if stream.readByte() else False elif code == BinaryObjectRegistry.CODE_STR: return stream.readString() elif code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES: return [readPrimitive(stream.readByte()) for _ in xrange(stream.readInt64())] else: assert False, "unknown code: " + str(code) if (code == BinaryObjectRegistry.CODE_NONE or code == BinaryObjectRegistry.CODE_INT or code == BinaryObjectRegistry.CODE_LONG or code == BinaryObjectRegistry.CODE_FLOAT or code == BinaryObjectRegistry.CODE_BOOL or code == BinaryObjectRegistry.CODE_STR or code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES): objectVisitor.definePrimitive(objectId, readPrimitive(code)) elif code == BinaryObjectRegistry.CODE_TUPLE: objectVisitor.defineTuple(objectId, readInt64s()) elif code == BinaryObjectRegistry.CODE_PACKED_HOMOGENOUS_DATA: dtype = readSimplePrimitive() packedBytes = stream.readString() objectVisitor.definePackedHomogenousData(objectId, TypeDescription.PackedHomogenousData(dtype, packedBytes)) elif code == BinaryObjectRegistry.CODE_LIST: objectVisitor.defineList(objectId, readInt64s()) elif code == BinaryObjectRegistry.CODE_FILE: path = stream.readString() text = stream.readString() objectVisitor.defineFile(objectId, text, path) elif code == BinaryObjectRegistry.CODE_DICT: objectVisitor.defineDict(objectId, readInt64s(), readInt64s()) elif code == BinaryObjectRegistry.CODE_REMOTE_PY_OBJECT: jsonRepresentation = json.loads(stream.readString()) objectVisitor.defineRemotePythonObject(objectId, convertJsonToObject(jsonRepresentation)) elif code == BinaryObjectRegistry.CODE_BUILTIN_EXCEPTION_INSTANCE: objectVisitor.defineBuiltinExceptionInstance(objectId, stream.readString(), stream.readInt64()) elif code == BinaryObjectRegistry.CODE_NAMED_SINGLETON: objectVisitor.defineNamedSingleton(objectId, stream.readString()) elif code == BinaryObjectRegistry.CODE_FUNCTION: objectVisitor.defineFunction(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds()) elif code == BinaryObjectRegistry.CODE_CLASS: objectVisitor.defineClass(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds(), readInt64s()) elif code == BinaryObjectRegistry.CODE_UNCONVERTIBLE: if stream.readByte() != 0: objectVisitor.defineUnconvertible(objectId, readStringTuple()) else: objectVisitor.defineUnconvertible(objectId, None) elif code == BinaryObjectRegistry.CODE_CLASS_INSTANCE: classId = stream.readInt64() classMembers = {} for _ in xrange(stream.readInt32()): memberName = stream.readString() classMembers[memberName] = stream.readInt64() objectVisitor.defineClassInstance(objectId, classId, classMembers) elif code == BinaryObjectRegistry.CODE_INSTANCE_METHOD: objectVisitor.defineInstanceMethod(objectId, stream.readInt64(), stream.readString()) elif code == BinaryObjectRegistry.CODE_WITH_BLOCK: scopes = readDottedScopeIds() objectVisitor.defineWithBlock(objectId, scopes, stream.readInt64(), stream.readInt32()) elif code == BinaryObjectRegistry.CODE_PY_ABORT_EXCEPTION: typename = stream.readString() argsId = stream.readInt64() objectVisitor.definePyAbortException(objectId, typename, argsId) elif code == BinaryObjectRegistry.CODE_STACKTRACE_AS_JSON: stackAsJson = stream.readString() objectVisitor.defineStacktrace(objectId, json.loads(stackAsJson)) elif code == BinaryObjectRegistry.CODE_UNRESOLVED_SYMBOL: varname = stream.readString() lineno = stream.readInt64() col_offset = stream.readInt64() objectVisitor.defineUnresolvedVarWithPosition(objectId, varname, lineno, col_offset) else: assert False, "BinaryStreamDeserializer.deserializeFromStream: unknown code: " + str(code)