Пример #1
0
    def test_cant_provide_mapping_for_named_singleton(self):
        mappings = PureImplementationMappings.PureImplementationMappings()

        #empty mappings work
        PyObjectWalker.PyObjectWalker(
            purePythonClassMapping=mappings,
            objectRegistry=None
            )

        mappings.addMapping(
            PureImplementationMapping.InstanceMapping(
                SomeRandomInstance(), SomeRandomInstance
                )
            )

        #an instance mapping doesn't cause an exception
        PyObjectWalker.PyObjectWalker(
            purePythonClassMapping=mappings,
            objectRegistry=None
            )

        self.assertTrue(UserWarning in NamedSingletons.pythonSingletonToName)

        mappings.addMapping(
            PureImplementationMapping.InstanceMapping(UserWarning, PureUserWarning)
            )

        #but this mapping doesnt
        with self.assertRaises(Exception):
            PyObjectWalker.PyObjectWalker(
                purePythonClassMapping=mappings,
                objectRegistry=None
                )
    def roundtripConvert(self, pyObject):
        mappings = PureImplementationMappings.PureImplementationMappings()
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)

        root_id = walker.walkPyObject(pyObject)
        binaryObjectRegistry.defineEndOfStream()

        data = binaryObjectRegistry.str()

        streamReader = PythonBinaryStreamToImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        streamReader.read(data)
        anObjAsImplval = streamReader.getObjectById(root_id)

        converter = PythonBinaryStreamFromImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        root_id, data = converter.write(anObjAsImplval)

        rehydrator = PythonObjectRehydrator(
            mappings, allowUserCodeModuleLevelLookups=False)

        return rehydrator.convertEncodedStringToPythonObject(data, root_id)
Пример #3
0
    def define(self, obj):
        """Create a remote representation of an object.

        Sends the specified object to the server and return a Future that resolves
        to a RemotePythonObject representing the object on the server.

        Args:
            obj: A python object to send

        Returns:
            A :class:`~Future.Future` that resolves to a :class:`~RemotePythonObject.RemotePythonObject`
            representing the object on the server.
        """

        self._raiseIfClosed()
        try:
            objectId = PyObjectWalker.PyObjectWalker(
                purePythonClassMapping=self.pureImplementationMappings,
                objectRegistry=self.objectRegistry).walkPyObject(obj)
        except PyObjectWalker.UnresolvedFreeVariableExceptionWithTrace as e:
            logging.error(
                "Converting UnresolvedFreeVariableExceptionWithTrace to PythonToForaConversionError:\n%s",
                traceback.format_exc())
            raise Exceptions.PythonToForaConversionError(e.message, e.trace)
        future = self._create_future()

        def onConverted(result):
            if not isinstance(result, Exception):
                result = RemotePythonObject.DefinedRemotePythonObject(
                    objectId, self)
            self._resolve_future(future, result)

        self.connection.convertObject(objectId, self.objectRegistry,
                                      onConverted)
        return future
Пример #4
0
def roundtripConvert(toConvert, vdm, allowUserCodeModuleLevelLookups=False):
    t0 = time.time()

    mappings = PureImplementationMappings.PureImplementationMappings()
    binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

    walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)

    objId = walker.walkPyObject(toConvert)

    binaryObjectRegistry.defineEndOfStream()

    t1 = time.time()

    registry = ObjectRegistry.ObjectRegistry()

    BinaryObjectRegistryDeserializer.deserializeFromString(
        binaryObjectRegistry.str(), registry, lambda x: x)

    t2 = time.time()

    objId, registry.objectIdToObjectDefinition = pickle.loads(
        pickle.dumps((objId, registry.objectIdToObjectDefinition), 2))

    t3 = time.time()

    converter = Converter.constructConverter(
        Converter.canonicalPurePythonModule(), vdm)
    anObjAsImplval = converter.convertDirectly(objId, registry)

    t4 = time.time()

    outputStream = BinaryObjectRegistry.BinaryObjectRegistry()

    root_id, needsLoad = converter.transformPyforaImplval(
        anObjAsImplval, outputStream,
        PyforaToJsonTransformer.ExtractVectorContents(vdm))

    needsLoad = False

    t5 = time.time()

    rehydrator = PythonObjectRehydrator(mappings,
                                        allowUserCodeModuleLevelLookups)

    finalResult = rehydrator.convertEncodedStringToPythonObject(
        outputStream.str(), root_id)

    t6 = time.time()

    return finalResult, {
        '0: walking': t1 - t0,
        '1: deserializeFromString': t2 - t1,
        '2: toImplval': t4 - t3,
        '3: serialze implVal': t5 - t4,
        '4: toPython': t6 - t5
    }
Пример #5
0
    def get_encoded_string_and_id(self, toConvert):
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(
            self.mappings,
            binaryObjectRegistry
            )

        objId = walker.walkPyObject(toConvert)

        binaryObjectRegistry.defineEndOfStream()

        return binaryObjectRegistry.str(), objId
Пример #6
0
    def _roundtripConvert(self, pyObject):
        t0 = time.time()

        mappings = PureImplementationMappings.PureImplementationMappings()
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(
            mappings,
            binaryObjectRegistry
            )

        root_id = walker.walkPyObject(pyObject)
        binaryObjectRegistry.defineEndOfStream()

        t1 = time.time()

        data = binaryObjectRegistry.str()

        streamReader = PythonBinaryStreamToImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        streamReader.read(data)
        anObjAsImplval = streamReader.getObjectById(root_id)
    
        t2 = time.time()

        converter = PythonBinaryStreamFromImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        root_id, data = converter.write(anObjAsImplval)

        t3 = time.time()

        rehydrator = PythonObjectRehydrator(
            mappings, allowUserCodeModuleLevelLookups=False)

        converted = rehydrator.convertEncodedStringToPythonObject(data, root_id)
        
        t4 = time.time()

        timings = {'py_to_binary': t1 - t0,
                   'binary_to_implVal': t2 - t1,
                   'implval_to_binary': t3 - t2,
                   'binary_to_python': t4 - t3}

        return converted, timings
Пример #7
0
    def test_conversion_metadata(self):
        for anInstance in [ThisIsAClass(), ThisIsAFunction]:
            mappings = PureImplementationMappings.PureImplementationMappings()

            binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

            walker = PyObjectWalker.PyObjectWalker(mappings,
                                                   binaryObjectRegistry)

            objId = walker.walkPyObject(anInstance)
            binaryObjectRegistry.defineEndOfStream()

            converter = Converter.constructConverter(
                Converter.canonicalPurePythonModule(), None)

            registry = ObjectRegistry.ObjectRegistry()

            BinaryObjectRegistryDeserializer.deserializeFromString(
                binaryObjectRegistry.str(), registry, lambda x: x)

            anObjAsImplval = converter.convertDirectly(objId, registry)

            stream = BinaryObjectRegistry.BinaryObjectRegistry()

            root_id, needsLoading = converter.transformPyforaImplval(
                anObjAsImplval, stream,
                PyforaToJsonTransformer.ExtractVectorContents(None))
            assert not needsLoading

            rehydrator = PythonObjectRehydrator(mappings, False)

            convertedInstance = rehydrator.convertEncodedStringToPythonObject(
                stream.str(), root_id)

            convertedInstanceModified = rehydrator.convertEncodedStringToPythonObject(
                stream.str().replace("return 100", "return 200"), root_id)

            if anInstance is ThisIsAFunction:
                self.assertEqual(anInstance(), 100)
                self.assertEqual(convertedInstance(), 100)
                self.assertEqual(convertedInstanceModified(), 200)
            else:
                self.assertEqual(anInstance.f(), 100)
                self.assertEqual(convertedInstance.f(), 100)
                self.assertEqual(convertedInstanceModified.f(), 200)
Пример #8
0
    def test_walking_unconvertible_module(self):
        mappings = PureImplementationMappings.PureImplementationMappings()
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)

        objId = walker.walkPyObject(ThisFunctionIsImpure)
        binaryObjectRegistry.defineEndOfStream()

        registry = ObjectRegistry.ObjectRegistry()
        BinaryObjectRegistryDeserializer.deserializeFromString(
            binaryObjectRegistry.str(), registry, lambda x: x)

        self.assertEqual(
            sorted(
                registry.objectIdToObjectDefinition[objId]\
                .freeVariableMemberAccessChainsToId.keys()
                ),
            ["multiprocessing"]
            )
Пример #9
0
    def test_basic_walking(self):
        x = 2
        y = 3

        def f():
            return x + g()

        def g():
            return y + f() + h()

        def h():
            pass

        testVisitor = TestVisitor1()
        walker = PyObjectWalker.PyObjectWalker(testVisitor)

        walker.walkPyObject(f)

        self.assertEqual(set(testVisitor.visitedValues),
                         set(['3', 'h', 'g', '2', 'f']))
Пример #10
0
    def executeOutOfProcessPythonCall(self, sock):
        mappings = PureImplementationMappings.PureImplementationMappings()
        mappings.load_pure_modules()

        rehydrator = PythonObjectRehydrator(
            mappings, allowUserCodeModuleLevelLookups=False)
        convertedInstance = rehydrator.readFileDescriptorToPythonObject(
            sock.fileno())

        result = convertedInstance()

        registry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(mappings, registry)

        objId = walker.walkPyObject(result)

        registry.defineEndOfStream()

        msg = registry.str() + struct.pack("<q", objId)
        Common.writeAllToFd(sock.fileno(), msg)
Пример #11
0
        def callback(aSocket):
            Common.writeAllToFd(aSocket.fileno(), Messages.MSG_OOPP_CALL)

            mappings = PureImplementationMappings.PureImplementationMappings()
            mappings.load_pure_modules()

            binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

            walker = PyObjectWalker.PyObjectWalker(mappings,
                                                   binaryObjectRegistry)

            objId = walker.walkPyObject(toCall)

            binaryObjectRegistry.defineEndOfStream()

            Common.writeAllToFd(aSocket.fileno(), binaryObjectRegistry.str())
            Common.writeAllToFd(aSocket.fileno(), struct.pack("<q", objId))

            rehydrator = PythonObjectRehydrator(
                mappings, allowUserCodeModuleLevelLookups=False)
            result.append(
                rehydrator.readFileDescriptorToPythonObject(aSocket.fileno()))
    def roundtripExecute(self, pyObject, *args):
        mappings = PureImplementationMappings.PureImplementationMappings()
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)

        ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)]
        binaryObjectRegistry.defineEndOfStream()

        data = binaryObjectRegistry.str()

        streamReader = PythonBinaryStreamToImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        streamReader.read(data)

        implVals = [streamReader.getObjectById(i) for i in ids]

        result = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
            ForaNative.ImplValContainer((implVals[0],
                                         ForaNative.makeSymbol("Call")) +
                                        tuple(implVals[1:])),
            InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1)

        self.assertTrue(result.isResult(), result)

        result = result.asResult.result

        converter = PythonBinaryStreamFromImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        root_id, data = converter.write(result)

        rehydrator = PythonObjectRehydrator(
            mappings, allowUserCodeModuleLevelLookups=False)

        return rehydrator.convertEncodedStringToPythonObject(data, root_id)
Пример #13
0
def walkPythonObject(pyObject, objectRegistry, purePythonClassMapping=None):
    """
    walk the python object `pyObject` in two passes,
    each time communicating with `objectRegistry`

    The first pass assigns ids to the nodes in `pyObject`.
    The second pass calls the appropriate `define*` methods on
    `objectRegistry` for the values it passes.

    In the end, return the object id of the converted value.

    purePythonClassMapping -- an instance of PureImplementationMappings
    """

    idVisitor = ObjectIdVisitor(objectRegistry=objectRegistry)

    pyObjectWalker = PyObjectWalker.PyObjectWalker(
        idVisitor, purePythonClassMapping=purePythonClassMapping)

    pyObjectWalker.walkPyObject(pyObject)

    objectVisitor = ObjectVisitor(
        objectRegistry, pyObjectIdToObjectId=idVisitor.pyObjectIdToObjectId)

    pyObjectWalker.resetWalkedNodes()
    pyObjectWalker.setVisitor(objectVisitor)

    pyObjectWalker.walkPyObject(pyObject)

    pyObject = pyObjectWalker.unwrapConvertedObject(pyObject)

    try:
        return objectVisitor.pyObjectIdToObjectId[id(pyObject)][1]
    except:
        logging.error("Failed with %s of type (%s)", pyObject, type(pyObject))
        raise