def roundtripConvert(self, pyObject): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) root_id = walker.walkPyObject(pyObject) binaryObjectRegistry.defineEndOfStream() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) anObjAsImplval = streamReader.getObjectById(root_id) converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(anObjAsImplval) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def test_cant_provide_mapping_for_named_singleton(self): mappings = PureImplementationMappings.PureImplementationMappings() #empty mappings work PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None ) mappings.addMapping( PureImplementationMapping.InstanceMapping( SomeRandomInstance(), SomeRandomInstance ) ) #an instance mapping doesn't cause an exception PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None ) self.assertTrue(UserWarning in NamedSingletons.pythonSingletonToName) mappings.addMapping( PureImplementationMapping.InstanceMapping(UserWarning, PureUserWarning) ) #but this mapping doesnt with self.assertRaises(Exception): PyObjectWalker.PyObjectWalker( purePythonClassMapping=mappings, objectRegistry=None )
def __init__(self, connection, pureImplementationMappings=None): self.connection = connection self.stayOpenOnExit = False self.pureImplementationMappings = \ pureImplementationMappings or PureImplementationMappings.PureImplementationMappings() self.objectRegistry = ObjectRegistry.ObjectRegistry() self.objectRehydrator = PythonObjectRehydrator.PythonObjectRehydrator( self.pureImplementationMappings) self.lock = threading.Lock()
def roundtripConvert(toConvert, vdm, allowUserCodeModuleLevelLookups=False): t0 = time.time() mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(toConvert) binaryObjectRegistry.defineEndOfStream() t1 = time.time() registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) t2 = time.time() objId, registry.objectIdToObjectDefinition = pickle.loads( pickle.dumps((objId, registry.objectIdToObjectDefinition), 2)) t3 = time.time() converter = Converter.constructConverter( Converter.canonicalPurePythonModule(), vdm) anObjAsImplval = converter.convertDirectly(objId, registry) t4 = time.time() outputStream = BinaryObjectRegistry.BinaryObjectRegistry() root_id, needsLoad = converter.transformPyforaImplval( anObjAsImplval, outputStream, PyforaToJsonTransformer.ExtractVectorContents(vdm)) needsLoad = False t5 = time.time() rehydrator = PythonObjectRehydrator(mappings, allowUserCodeModuleLevelLookups) finalResult = rehydrator.convertEncodedStringToPythonObject( outputStream.str(), root_id) t6 = time.time() return finalResult, { '0: walking': t1 - t0, '1: deserializeFromString': t2 - t1, '2: toImplval': t4 - t3, '3: serialze implVal': t5 - t4, '4: toPython': t6 - t5 }
def setUp(self): self.excludeList = ["staticmethod", "property", "__inline_fora"] def is_pureMapping_call(node): return isinstance(node, ast.Call) and \ isinstance(node.func, ast.Name) and \ node.func.id == 'pureMapping' self.excludePredicateFun = is_pureMapping_call self.mappings = PureImplementationMappings.PureImplementationMappings()
def __init__(self, visitor, purePythonClassMapping=None): self.walkedNodes = dict() self.visitor = visitor if purePythonClassMapping is None: purePythonClassMapping = PureImplementationMappings.PureImplementationMappings( ) self.purePythonClassMapping = purePythonClassMapping self._convertedObjectCache = dict() self._fileTextCache = dict()
def __init__(self, purePythonClassMapping, objectRegistry): if purePythonClassMapping is None: purePythonClassMapping = \ PureImplementationMappings.PureImplementationMappings() for singleton in NamedSingletons.pythonSingletonToName: if purePythonClassMapping.canMap(singleton): raise UserWarning("You provided a mapping that applies to %s, " "which already has a direct mapping" % singleton) self._purePythonClassMapping = purePythonClassMapping self._convertedObjectCache = {} self._pyObjectIdToObjectId = {} self._objectRegistry = objectRegistry
def _roundtripConvert(self, pyObject): t0 = time.time() mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker( mappings, binaryObjectRegistry ) root_id = walker.walkPyObject(pyObject) binaryObjectRegistry.defineEndOfStream() t1 = time.time() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) anObjAsImplval = streamReader.getObjectById(root_id) t2 = time.time() converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(anObjAsImplval) t3 = time.time() rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) converted = rehydrator.convertEncodedStringToPythonObject(data, root_id) t4 = time.time() timings = {'py_to_binary': t1 - t0, 'binary_to_implVal': t2 - t1, 'implval_to_binary': t3 - t2, 'binary_to_python': t4 - t3} return converted, timings
def test_conversion_metadata(self): for anInstance in [ThisIsAClass(), ThisIsAFunction]: mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(anInstance) binaryObjectRegistry.defineEndOfStream() converter = Converter.constructConverter( Converter.canonicalPurePythonModule(), None) registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) anObjAsImplval = converter.convertDirectly(objId, registry) stream = BinaryObjectRegistry.BinaryObjectRegistry() root_id, needsLoading = converter.transformPyforaImplval( anObjAsImplval, stream, PyforaToJsonTransformer.ExtractVectorContents(None)) assert not needsLoading rehydrator = PythonObjectRehydrator(mappings, False) convertedInstance = rehydrator.convertEncodedStringToPythonObject( stream.str(), root_id) convertedInstanceModified = rehydrator.convertEncodedStringToPythonObject( stream.str().replace("return 100", "return 200"), root_id) if anInstance is ThisIsAFunction: self.assertEqual(anInstance(), 100) self.assertEqual(convertedInstance(), 100) self.assertEqual(convertedInstanceModified(), 200) else: self.assertEqual(anInstance.f(), 100) self.assertEqual(convertedInstance.f(), 100) self.assertEqual(convertedInstanceModified.f(), 200)
def test_walking_unconvertible_module(self): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(ThisFunctionIsImpure) binaryObjectRegistry.defineEndOfStream() registry = ObjectRegistry.ObjectRegistry() BinaryObjectRegistryDeserializer.deserializeFromString( binaryObjectRegistry.str(), registry, lambda x: x) self.assertEqual( sorted( registry.objectIdToObjectDefinition[objId]\ .freeVariableMemberAccessChainsToId.keys() ), ["multiprocessing"] )
def executeOutOfProcessPythonCall(self, sock): mappings = PureImplementationMappings.PureImplementationMappings() mappings.load_pure_modules() rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) convertedInstance = rehydrator.readFileDescriptorToPythonObject( sock.fileno()) result = convertedInstance() registry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, registry) objId = walker.walkPyObject(result) registry.defineEndOfStream() msg = registry.str() + struct.pack("<q", objId) Common.writeAllToFd(sock.fileno(), msg)
def callback(aSocket): Common.writeAllToFd(aSocket.fileno(), Messages.MSG_OOPP_CALL) mappings = PureImplementationMappings.PureImplementationMappings() mappings.load_pure_modules() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) objId = walker.walkPyObject(toCall) binaryObjectRegistry.defineEndOfStream() Common.writeAllToFd(aSocket.fileno(), binaryObjectRegistry.str()) Common.writeAllToFd(aSocket.fileno(), struct.pack("<q", objId)) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) result.append( rehydrator.readFileDescriptorToPythonObject(aSocket.fileno()))
def roundtripExecute(self, pyObject, *args): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)] binaryObjectRegistry.defineEndOfStream() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) implVals = [streamReader.getObjectById(i) for i in ids] result = InMemoryCumulusSimulation.computeUsingSeveralWorkers( ForaNative.ImplValContainer((implVals[0], ForaNative.makeSymbol("Call")) + tuple(implVals[1:])), InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1) self.assertTrue(result.isResult(), result) result = result.asResult.result converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(result) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) return rehydrator.convertEncodedStringToPythonObject(data, root_id)
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pyfora.PureImplementationMappings as PureImplementationMappings import pyfora.BuiltinPureImplementationMappings as BuiltinPureImplementationMappings import pyfora.typeConverters.Numpy as Numpy defaultMapping_ = PureImplementationMappings.PureImplementationMappings() defaultMapping_.addMapping(Numpy.PurePythonNumpyArrayMapping()) for mapping in BuiltinPureImplementationMappings.generateMappings(): defaultMapping_.addMapping(mapping) def getMappings(): return defaultMapping_ def addMapping(mapping): """Register an instance of PureImplementationMapping with the default mapping model. This is the primary way that users can register mappings for libraries that are not part of the default pyfora mapping libraries. """
def setUp(self): self.mappings = PureImplementationMappings.PureImplementationMappings()
def test_PyObjectWalker_TypeErrors(self): mappings = PureImplementationMappings.PureImplementationMappings() with self.assertRaises(TypeError): not_an_objectregistry = 2 PyObjectWalker(mappings, not_an_objectregistry)
# limitations under the License. import ast import sys import pyfora.PureImplementationMappings as PureImplementationMappings exclude_list = ["staticmethod", "property", "__inline_fora"] def exclude_predicate_fun(node): return isinstance(node, ast.Call) and \ isinstance(node.func, ast.Name) and \ node.func.id == 'pureMapping' mappings = PureImplementationMappings.PureImplementationMappings() def terminal_value_filter(terminalValue): return not mappings.isOpaqueModule(terminalValue) def pythonTracebackToJson(stacktrace): if stacktrace is None: return None res = [] while stacktrace is not None: filename = stacktrace.tb_frame.f_code.co_filename lineno = stacktrace.tb_lineno