def roundtripConvert(self, pyObject):
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)
root_id = walker.walkPyObject(pyObject)
binaryObjectRegistry.defineEndOfStream()
data = binaryObjectRegistry.str()
streamReader = PythonBinaryStreamToImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
streamReader.read(data)
anObjAsImplval = streamReader.getObjectById(root_id)
converter = PythonBinaryStreamFromImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
root_id, data = converter.write(anObjAsImplval)
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def test_cant_provide_mapping_for_named_singleton(self):
mappings = PureImplementationMappings.PureImplementationMappings()
#empty mappings work
PyObjectWalker.PyObjectWalker(
purePythonClassMapping=mappings,
objectRegistry=None
)
mappings.addMapping(
PureImplementationMapping.InstanceMapping(
SomeRandomInstance(), SomeRandomInstance
)
)
#an instance mapping doesn't cause an exception
PyObjectWalker.PyObjectWalker(
purePythonClassMapping=mappings,
objectRegistry=None
)
self.assertTrue(UserWarning in NamedSingletons.pythonSingletonToName)
mappings.addMapping(
PureImplementationMapping.InstanceMapping(UserWarning, PureUserWarning)
)
#but this mapping doesnt
with self.assertRaises(Exception):
PyObjectWalker.PyObjectWalker(
purePythonClassMapping=mappings,
objectRegistry=None
)
def __init__(self, connection, pureImplementationMappings=None):
self.connection = connection
self.stayOpenOnExit = False
self.pureImplementationMappings = \
pureImplementationMappings or PureImplementationMappings.PureImplementationMappings()
self.objectRegistry = ObjectRegistry.ObjectRegistry()
self.objectRehydrator = PythonObjectRehydrator.PythonObjectRehydrator(
self.pureImplementationMappings)
self.lock = threading.Lock()
def roundtripConvert(toConvert, vdm, allowUserCodeModuleLevelLookups=False):
t0 = time.time()
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)
objId = walker.walkPyObject(toConvert)
binaryObjectRegistry.defineEndOfStream()
t1 = time.time()
registry = ObjectRegistry.ObjectRegistry()
BinaryObjectRegistryDeserializer.deserializeFromString(
binaryObjectRegistry.str(), registry, lambda x: x)
t2 = time.time()
objId, registry.objectIdToObjectDefinition = pickle.loads(
pickle.dumps((objId, registry.objectIdToObjectDefinition), 2))
t3 = time.time()
converter = Converter.constructConverter(
Converter.canonicalPurePythonModule(), vdm)
anObjAsImplval = converter.convertDirectly(objId, registry)
t4 = time.time()
outputStream = BinaryObjectRegistry.BinaryObjectRegistry()
root_id, needsLoad = converter.transformPyforaImplval(
anObjAsImplval, outputStream,
PyforaToJsonTransformer.ExtractVectorContents(vdm))
needsLoad = False
t5 = time.time()
rehydrator = PythonObjectRehydrator(mappings,
allowUserCodeModuleLevelLookups)
finalResult = rehydrator.convertEncodedStringToPythonObject(
outputStream.str(), root_id)
t6 = time.time()
return finalResult, {
'0: walking': t1 - t0,
'1: deserializeFromString': t2 - t1,
'2: toImplval': t4 - t3,
'3: serialze implVal': t5 - t4,
'4: toPython': t6 - t5
}
def setUp(self):
self.excludeList = ["staticmethod", "property", "__inline_fora"]
def is_pureMapping_call(node):
return isinstance(node, ast.Call) and \
isinstance(node.func, ast.Name) and \
node.func.id == 'pureMapping'
self.excludePredicateFun = is_pureMapping_call
self.mappings = PureImplementationMappings.PureImplementationMappings()
def __init__(self, visitor, purePythonClassMapping=None):
self.walkedNodes = dict()
self.visitor = visitor
if purePythonClassMapping is None:
purePythonClassMapping = PureImplementationMappings.PureImplementationMappings(
)
self.purePythonClassMapping = purePythonClassMapping
self._convertedObjectCache = dict()
self._fileTextCache = dict()
def __init__(self, purePythonClassMapping, objectRegistry):
if purePythonClassMapping is None:
purePythonClassMapping = \
PureImplementationMappings.PureImplementationMappings()
for singleton in NamedSingletons.pythonSingletonToName:
if purePythonClassMapping.canMap(singleton):
raise UserWarning("You provided a mapping that applies to %s, "
"which already has a direct mapping" %
singleton)
self._purePythonClassMapping = purePythonClassMapping
self._convertedObjectCache = {}
self._pyObjectIdToObjectId = {}
self._objectRegistry = objectRegistry
def _roundtripConvert(self, pyObject):
t0 = time.time()
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(
mappings,
binaryObjectRegistry
)
root_id = walker.walkPyObject(pyObject)
binaryObjectRegistry.defineEndOfStream()
t1 = time.time()
data = binaryObjectRegistry.str()
streamReader = PythonBinaryStreamToImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
streamReader.read(data)
anObjAsImplval = streamReader.getObjectById(root_id)
t2 = time.time()
converter = PythonBinaryStreamFromImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
root_id, data = converter.write(anObjAsImplval)
t3 = time.time()
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
converted = rehydrator.convertEncodedStringToPythonObject(data, root_id)
t4 = time.time()
timings = {'py_to_binary': t1 - t0,
'binary_to_implVal': t2 - t1,
'implval_to_binary': t3 - t2,
'binary_to_python': t4 - t3}
return converted, timings
def test_conversion_metadata(self):
for anInstance in [ThisIsAClass(), ThisIsAFunction]:
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings,
binaryObjectRegistry)
objId = walker.walkPyObject(anInstance)
binaryObjectRegistry.defineEndOfStream()
converter = Converter.constructConverter(
Converter.canonicalPurePythonModule(), None)
registry = ObjectRegistry.ObjectRegistry()
BinaryObjectRegistryDeserializer.deserializeFromString(
binaryObjectRegistry.str(), registry, lambda x: x)
anObjAsImplval = converter.convertDirectly(objId, registry)
stream = BinaryObjectRegistry.BinaryObjectRegistry()
root_id, needsLoading = converter.transformPyforaImplval(
anObjAsImplval, stream,
PyforaToJsonTransformer.ExtractVectorContents(None))
assert not needsLoading
rehydrator = PythonObjectRehydrator(mappings, False)
convertedInstance = rehydrator.convertEncodedStringToPythonObject(
stream.str(), root_id)
convertedInstanceModified = rehydrator.convertEncodedStringToPythonObject(
stream.str().replace("return 100", "return 200"), root_id)
if anInstance is ThisIsAFunction:
self.assertEqual(anInstance(), 100)
self.assertEqual(convertedInstance(), 100)
self.assertEqual(convertedInstanceModified(), 200)
else:
self.assertEqual(anInstance.f(), 100)
self.assertEqual(convertedInstance.f(), 100)
self.assertEqual(convertedInstanceModified.f(), 200)
def test_walking_unconvertible_module(self):
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)
objId = walker.walkPyObject(ThisFunctionIsImpure)
binaryObjectRegistry.defineEndOfStream()
registry = ObjectRegistry.ObjectRegistry()
BinaryObjectRegistryDeserializer.deserializeFromString(
binaryObjectRegistry.str(), registry, lambda x: x)
self.assertEqual(
sorted(
registry.objectIdToObjectDefinition[objId]\
.freeVariableMemberAccessChainsToId.keys()
),
["multiprocessing"]
)
def executeOutOfProcessPythonCall(self, sock):
mappings = PureImplementationMappings.PureImplementationMappings()
mappings.load_pure_modules()
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
convertedInstance = rehydrator.readFileDescriptorToPythonObject(
sock.fileno())
result = convertedInstance()
registry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, registry)
objId = walker.walkPyObject(result)
registry.defineEndOfStream()
msg = registry.str() + struct.pack("<q", objId)
Common.writeAllToFd(sock.fileno(), msg)
def callback(aSocket):
Common.writeAllToFd(aSocket.fileno(), Messages.MSG_OOPP_CALL)
mappings = PureImplementationMappings.PureImplementationMappings()
mappings.load_pure_modules()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings,
binaryObjectRegistry)
objId = walker.walkPyObject(toCall)
binaryObjectRegistry.defineEndOfStream()
Common.writeAllToFd(aSocket.fileno(), binaryObjectRegistry.str())
Common.writeAllToFd(aSocket.fileno(), struct.pack("<q", objId))
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
result.append(
rehydrator.readFileDescriptorToPythonObject(aSocket.fileno()))
def roundtripExecute(self, pyObject, *args):
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)
ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)]
binaryObjectRegistry.defineEndOfStream()
data = binaryObjectRegistry.str()
streamReader = PythonBinaryStreamToImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
streamReader.read(data)
implVals = [streamReader.getObjectById(i) for i in ids]
result = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
ForaNative.ImplValContainer((implVals[0],
ForaNative.makeSymbol("Call")) +
tuple(implVals[1:])),
InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1)
self.assertTrue(result.isResult(), result)
result = result.asResult.result
converter = PythonBinaryStreamFromImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
root_id, data = converter.write(result)
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
return rehydrator.convertEncodedStringToPythonObject(data, root_id)
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pyfora.PureImplementationMappings as PureImplementationMappings
import pyfora.BuiltinPureImplementationMappings as BuiltinPureImplementationMappings
import pyfora.typeConverters.Numpy as Numpy
defaultMapping_ = PureImplementationMappings.PureImplementationMappings()
defaultMapping_.addMapping(Numpy.PurePythonNumpyArrayMapping())
for mapping in BuiltinPureImplementationMappings.generateMappings():
defaultMapping_.addMapping(mapping)
def getMappings():
return defaultMapping_
def addMapping(mapping):
"""Register an instance of PureImplementationMapping with the default mapping model.
This is the primary way that users can register mappings for libraries that are not part
of the default pyfora mapping libraries.
"""
def setUp(self):
self.mappings = PureImplementationMappings.PureImplementationMappings()
def test_PyObjectWalker_TypeErrors(self):
mappings = PureImplementationMappings.PureImplementationMappings()
with self.assertRaises(TypeError):
not_an_objectregistry = 2
PyObjectWalker(mappings, not_an_objectregistry)
# limitations under the License.
import ast
import sys
import pyfora.PureImplementationMappings as PureImplementationMappings
exclude_list = ["staticmethod", "property", "__inline_fora"]
def exclude_predicate_fun(node):
return isinstance(node, ast.Call) and \
isinstance(node.func, ast.Name) and \
node.func.id == 'pureMapping'
mappings = PureImplementationMappings.PureImplementationMappings()
def terminal_value_filter(terminalValue):
return not mappings.isOpaqueModule(terminalValue)
def pythonTracebackToJson(stacktrace):
if stacktrace is None:
return None
res = []
while stacktrace is not None:
filename = stacktrace.tb_frame.f_code.co_filename
lineno = stacktrace.tb_lineno
