def test_cant_provide_mapping_for_named_singleton(self): mappings = PureImplementationMappings.PureImplementationMappings() #empty mappings work PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None ) mappings.addMapping( PureImplementationMapping.InstanceMapping( SomeRandomInstance(), SomeRandomInstance ) ) #an instance mapping doesn't cause an exception PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None ) self.assertTrue(UserWarning in NamedSingletons.pythonSingletonToName) mappings.addMapping( PureImplementationMapping.InstanceMapping(UserWarning, PureUserWarning) ) #but this mapping doesnt with self.assertRaises(Exception): PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None )
def roundtripConvert(self, pyObject): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) root_id = walker.walkPyObject(pyObject) binaryObjectRegistry.defineEndOfStream() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) anObjAsImplval = streamReader.getObjectById(root_id) converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(anObjAsImplval) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def define(self, obj): """Create a remote representation of an object. Sends the specified object to the server and return a Future that resolves to a RemotePythonObject representing the object on the server. Args: obj: A python object to send Returns: A :class:`~Future.Future` that resolves to a :class:`~RemotePythonObject.RemotePythonObject` representing the object on the server. """ self._raiseIfClosed() try: objectId = PyObjectWalker.PyObjectWalker( purePythonClassMapping=self.pureImplementationMappings, objectRegistry=self.objectRegistry).walkPyObject(obj) except PyObjectWalker.UnresolvedFreeVariableExceptionWithTrace as e: logging.error( "Converting UnresolvedFreeVariableExceptionWithTrace to PythonToForaConversionError:\n%s", traceback.format_exc()) raise Exceptions.PythonToForaConversionError(e.message, e.trace) future = self._create_future() def onConverted(result): if not isinstance(result, Exception): result = RemotePythonObject.DefinedRemotePythonObject( objectId, self) self._resolve_future(future, result) self.connection.convertObject(objectId, self.objectRegistry, onConverted) return future
def roundtripConvert(toConvert, vdm, allowUserCodeModuleLevelLookups=False): t0 = time.time() mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(toConvert) binaryObjectRegistry.defineEndOfStream() t1 = time.time() registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) t2 = time.time() objId, registry.objectIdToObjectDefinition = pickle.loads( pickle.dumps((objId, registry.objectIdToObjectDefinition), 2)) t3 = time.time() converter = Converter.constructConverter( Converter.canonicalPurePythonModule(), vdm) anObjAsImplval = converter.convertDirectly(objId, registry) t4 = time.time() outputStream = BinaryObjectRegistry.BinaryObjectRegistry() root_id, needsLoad = converter.transformPyforaImplval( anObjAsImplval, outputStream, PyforaToJsonTransformer.ExtractVectorContents(vdm)) needsLoad = False t5 = time.time() rehydrator = PythonObjectRehydrator(mappings, allowUserCodeModuleLevelLookups) finalResult = rehydrator.convertEncodedStringToPythonObject( outputStream.str(), root_id) t6 = time.time() return finalResult, { '0: walking': t1 - t0, '1: deserializeFromString': t2 - t1, '2: toImplval': t4 - t3, '3: serialze implVal': t5 - t4, '4: toPython': t6 - t5 }
def get_encoded_string_and_id(self, toConvert): binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker( self.mappings, binaryObjectRegistry ) objId = walker.walkPyObject(toConvert) binaryObjectRegistry.defineEndOfStream() return binaryObjectRegistry.str(), objId
def _roundtripConvert(self, pyObject): t0 = time.time() mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker( mappings, binaryObjectRegistry ) root_id = walker.walkPyObject(pyObject) binaryObjectRegistry.defineEndOfStream() t1 = time.time() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) anObjAsImplval = streamReader.getObjectById(root_id) t2 = time.time() converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(anObjAsImplval) t3 = time.time() rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) converted = rehydrator.convertEncodedStringToPythonObject(data, root_id) t4 = time.time() timings = {'py_to_binary': t1 - t0, 'binary_to_implVal': t2 - t1, 'implval_to_binary': t3 - t2, 'binary_to_python': t4 - t3} return converted, timings
def test_conversion_metadata(self): for anInstance in [ThisIsAClass(), ThisIsAFunction]: mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(anInstance) binaryObjectRegistry.defineEndOfStream() converter = Converter.constructConverter( Converter.canonicalPurePythonModule(), None) registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) anObjAsImplval = converter.convertDirectly(objId, registry) stream = BinaryObjectRegistry.BinaryObjectRegistry() root_id, needsLoading = converter.transformPyforaImplval( anObjAsImplval, stream, PyforaToJsonTransformer.ExtractVectorContents(None)) assert not needsLoading rehydrator = PythonObjectRehydrator(mappings, False) convertedInstance = rehydrator.convertEncodedStringToPythonObject( stream.str(), root_id) convertedInstanceModified = rehydrator.convertEncodedStringToPythonObject( stream.str().replace("return 100", "return 200"), root_id) if anInstance is ThisIsAFunction: self.assertEqual(anInstance(), 100) self.assertEqual(convertedInstance(), 100) self.assertEqual(convertedInstanceModified(), 200) else: self.assertEqual(anInstance.f(), 100) self.assertEqual(convertedInstance.f(), 100) self.assertEqual(convertedInstanceModified.f(), 200)
def test_walking_unconvertible_module(self): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(ThisFunctionIsImpure) binaryObjectRegistry.defineEndOfStream() registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) self.assertEqual( sorted( registry.objectIdToObjectDefinition[objId]\ .freeVariableMemberAccessChainsToId.keys() ), ["multiprocessing"] )
def test_basic_walking(self): x = 2 y = 3 def f(): return x + g() def g(): return y + f() + h() def h(): pass testVisitor = TestVisitor1() walker = PyObjectWalker.PyObjectWalker(testVisitor) walker.walkPyObject(f) self.assertEqual(set(testVisitor.visitedValues), set(['3', 'h', 'g', '2', 'f']))
def executeOutOfProcessPythonCall(self, sock): mappings = PureImplementationMappings.PureImplementationMappings() mappings.load_pure_modules() rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) convertedInstance = rehydrator.readFileDescriptorToPythonObject( sock.fileno()) result = convertedInstance() registry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, registry) objId = walker.walkPyObject(result) registry.defineEndOfStream() msg = registry.str() + struct.pack("<q", objId) Common.writeAllToFd(sock.fileno(), msg)
def callback(aSocket): Common.writeAllToFd(aSocket.fileno(), Messages.MSG_OOPP_CALL) mappings = PureImplementationMappings.PureImplementationMappings() mappings.load_pure_modules() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(toCall) binaryObjectRegistry.defineEndOfStream() Common.writeAllToFd(aSocket.fileno(), binaryObjectRegistry.str()) Common.writeAllToFd(aSocket.fileno(), struct.pack("<q", objId)) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) result.append( rehydrator.readFileDescriptorToPythonObject(aSocket.fileno()))
def roundtripExecute(self, pyObject, *args): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)] binaryObjectRegistry.defineEndOfStream() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) implVals = [streamReader.getObjectById(i) for i in ids] result = InMemoryCumulusSimulation.computeUsingSeveralWorkers( ForaNative.ImplValContainer((implVals[0], ForaNative.makeSymbol("Call")) + tuple(implVals[1:])), InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1) self.assertTrue(result.isResult(), result) result = result.asResult.result converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(result) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def walkPythonObject(pyObject, objectRegistry, purePythonClassMapping=None): """ walk the python object `pyObject` in two passes, each time communicating with `objectRegistry` The first pass assigns ids to the nodes in `pyObject`. The second pass calls the appropriate `define*` methods on `objectRegistry` for the values it passes. In the end, return the object id of the converted value. purePythonClassMapping -- an instance of PureImplementationMappings """ idVisitor = ObjectIdVisitor(objectRegistry=objectRegistry) pyObjectWalker = PyObjectWalker.PyObjectWalker( idVisitor, purePythonClassMapping=purePythonClassMapping) pyObjectWalker.walkPyObject(pyObject) objectVisitor = ObjectVisitor( objectRegistry, pyObjectIdToObjectId=idVisitor.pyObjectIdToObjectId) pyObjectWalker.resetWalkedNodes() pyObjectWalker.setVisitor(objectVisitor) pyObjectWalker.walkPyObject(pyObject) pyObject = pyObjectWalker.unwrapConvertedObject(pyObject) try: return objectVisitor.pyObjectIdToObjectId[id(pyObject)][1] except: logging.error("Failed with %s of type (%s)", pyObject, type(pyObject)) raise