def test_simple_serialization_and_deserialization(self):
typeDescription = TypeDescription.Primitive(23.4)
self.assertDeserializationSerializationSucceeded(typeDescription)
typeDescription = TypeDescription.Tuple([1, 2, 3, 4, 5])
self.assertDeserializationSerializationSucceeded(typeDescription)
typeDescription = TypeDescription.ClassDefinition("source text here", ['test'])
self.assertDeserializationSerializationSucceeded(typeDescription)
def mapPythonInstanceToPyforaInstance(self, numpyArray):
flattened = numpyArray.flatten()
dtypeAsPrim = TypeDescription.dtypeToPrimitive(numpyArray.dtype)
if dtypeAsPrim is not None:
return PurePythonNumpyArray(
numpyArray.shape,
TypeDescription.PackedHomogenousData(dtypeAsPrim,
flattened.tostring()))
else:
return PurePythonNumpyArray(numpyArray.shape, flattened.tolist())
def test_numpy_dtype_conversion(self):
for array in [
numpy.array([1.0, 2.0]),
numpy.zeros(2, dtype=[('f0', '<f8'), ('f1', '<f8')]),
numpy.array([1]),
numpy.array([False])
]:
dt = array.dtype
dt2 = TypeDescription.primitiveToDtype(
TypeDescription.dtypeToPrimitive(array.dtype))
self.assertEqual(str(dt.descr), str(dt2.descr))
def convert(self, objectId, objectRegistry, callback):
dependencyGraph = objectRegistry.computeDependencyGraph(objectId)
objectIdToObjectDefinition = {
objId:
TypeDescription.serialize(objectRegistry.getDefinition(objId))
for objId in dependencyGraph.iterkeys()
}
def onSuccess(message):
if 'isException' not in message:
callback(objectId)
else:
callback(
Exceptions.PythonToForaConversionError(
str(message['message']), message['trace']))
self.remoteConverter.convert(
{
'objectId': objectId,
'objectIdToObjectDefinition': objectIdToObjectDefinition
}, {
'onSuccess':
onSuccess,
'onFailure':
lambda err: callback(
Exceptions.PythonToForaConversionError(err))
})
def defineClassInstance(self, objectId, classId,
classMemberNameToClassMemberId):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.ClassInstanceDescription(
classId=classId,
classMemberNameToClassMemberId=classMemberNameToClassMemberId
)
def defineWithBlock(self, objectId, freeVariableMemberAccessChainsToId,
sourceFileId, lineNumber):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.WithBlockDescription(
freeVariableMemberAccessChainsToId,
sourceFileId,
lineNumber
)
def defineFunction(self, objectId, sourceFileId, lineNumber, scopeIds):
"""
scopeIds: a dict freeVariableMemberAccessChain -> id
"""
freeVariableMemberAccessChainsToId = \
self._processFreeVariableMemberAccessChainResolution(scopeIds)
self.objectIdToObjectDefinition[objectId] = TypeDescription.FunctionDefinition(
sourceFileId=sourceFileId,
lineNumber=lineNumber,
freeVariableMemberAccessChainsToId=freeVariableMemberAccessChainsToId
)
def defineClass(self, objectId, sourceFileId, lineNumber, scopeIds, baseClassIds):
"""
scopeIds: a dict freeVariableMemberAccessChain -> id
baseClassIds: a list of ids representing (immediate) base classes
"""
freeVariableMemberAccessChainsToId = \
self._processFreeVariableMemberAccessChainResolution(
scopeIds
)
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.ClassDefinition(
sourceFileId=sourceFileId,
lineNumber=lineNumber,
freeVariableMemberAccessChainsToId=freeVariableMemberAccessChainsToId,
baseClassIds=baseClassIds
)
def _computeDependentIds(self, objectId):
objectDefinition = self.objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or \
isinstance(objectDefinition,
(TypeDescription.File, TypeDescription.RemotePythonObject,
TypeDescription.NamedSingleton, list,
TypeDescription.Unconvertible,
TypeDescription.PackedHomogenousData,
TypeDescription.UnresolvedVarWithPosition
)):
return []
elif isinstance(objectDefinition,
(TypeDescription.BuiltinExceptionInstance)):
return [objectDefinition.argsId]
elif isinstance(objectDefinition,
(TypeDescription.List, TypeDescription.Tuple)):
return objectDefinition.memberIds
elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition)):
tr = objectDefinition.freeVariableMemberAccessChainsToId.values()
tr.append(objectDefinition.sourceFileId)
return tr
elif isinstance(objectDefinition, TypeDescription.InstanceMethod):
return [objectDefinition.instanceId]
elif isinstance(objectDefinition,
TypeDescription.ClassInstanceDescription):
tr = [objectDefinition.classId]
tr.extend(self._computeDependentIds(objectDefinition.classId))
classMemberIds = \
objectDefinition.classMemberNameToClassMemberId.values()
tr.extend(classMemberIds)
return tr
elif isinstance(objectDefinition, TypeDescription.Dict):
return objectDefinition.keyIds + objectDefinition.valueIds
elif isinstance(objectDefinition,
TypeDescription.WithBlockDescription):
tr = objectDefinition.freeVariableMemberAccessChainsToId.values()
tr.append(objectDefinition.sourceFileId)
return tr
else:
assert False, "don't know what to do with %s" % type(
objectDefinition)
def _computeDependentIds(self, objectId):
objectDefinition = self.objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or \
isinstance(objectDefinition,
(TypeDescription.File, TypeDescription.RemotePythonObject,
TypeDescription.NamedSingleton, list,
TypeDescription.Unconvertible)):
return []
elif isinstance(objectDefinition, (TypeDescription.BuiltinExceptionInstance)):
return [objectDefinition.argsId]
elif isinstance(objectDefinition, (TypeDescription.List, TypeDescription.Tuple)):
return objectDefinition.memberIds
elif isinstance(objectDefinition,
(TypeDescription.FunctionDefinition, TypeDescription.ClassDefinition)):
tr = objectDefinition.freeVariableMemberAccessChainsToId.values()
tr.append(objectDefinition.sourceFileId)
return tr
elif isinstance(objectDefinition, TypeDescription.InstanceMethod):
return [objectDefinition.instanceId]
elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription):
tr = [objectDefinition.classId]
tr.extend(
self._computeDependentIds(
objectDefinition.classId
)
)
classMemberIds = \
objectDefinition.classMemberNameToClassMemberId.values()
tr.extend(classMemberIds)
return tr
elif isinstance(objectDefinition, TypeDescription.Dict):
return objectDefinition.keyIds + objectDefinition.valueIds
elif isinstance(objectDefinition, TypeDescription.WithBlockDescription):
tr = objectDefinition.freeVariableMemberAccessChainsToId.values()
tr.append(objectDefinition.sourceFileId)
return tr
else:
assert False, "don't know what to do with %s" % type(objectDefinition)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition):
objectDefinition = objectIdToObjectDefinition[objectId]
logging.info("ObjectDefinition: %s", objectDefinition)
if TypeDescription.isPrimitive(objectDefinition) or isinstance(
objectDefinition, list):
return self.convertPrimitive(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
return self.convertRemotePythonObject(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
return self.convertNamedSingleton(objectDefinition)
elif isinstance(objectDefinition,
TypeDescription.BuiltinExceptionInstance):
return self.convertBuiltinExceptionInstance(objectDefinition)
elif isinstance(objectDefinition,
(TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition,
TypeDescription.ClassInstanceDescription,
TypeDescription.InstanceMethod)):
return (self.convertObjectWithDependencies(
objectId, dependencyGraph, objectIdToObjectDefinition))
elif isinstance(objectDefinition, TypeDescription.List):
return (self.convertList(objectId, dependencyGraph,
objectIdToObjectDefinition))
elif isinstance(objectDefinition, TypeDescription.Tuple):
return (self.convertTuple(objectId, dependencyGraph,
objectIdToObjectDefinition))
elif isinstance(objectDefinition, TypeDescription.Dict):
return (self.convertDict(objectId, objectDefinition,
dependencyGraph,
objectIdToObjectDefinition))
elif isinstance(objectDefinition,
TypeDescription.WithBlockDescription):
return (self.convertObjectWithDependencies(
objectId, dependencyGraph, objectIdToObjectDefinition))
else:
raise pyfora.PythonToForaConversionError(
"don't know how to convert %s of type %s" %
(objectDefinition, type(objectDefinition)))
def convert(self, objectId, objectRegistry, callback):
dependencyGraph = objectRegistry.computeDependencyGraph(objectId)
objectIdToObjectDefinition = {
objId: TypeDescription.serialize(objectRegistry.getDefinition(objId))
for objId in dependencyGraph.iterkeys()
}
def onSuccess(message):
if 'isException' not in message:
callback(objectId)
else:
callback(Exceptions.PythonToForaConversionError(str(message['message']), message['trace']))
self.remoteConverter.convert(
{
'objectId': objectId,
'objectIdToObjectDefinition': objectIdToObjectDefinition
},
{
'onSuccess': onSuccess,
'onFailure': lambda err: callback(Exceptions.PythonToForaConversionError(err))
})
def convert(self, objectId, objectIdToObjectDefinition):
import pyfora.TypeDescription as TypeDescription
import pyfora.Exceptions as PyforaExceptions
result = [None]
def onConverted(r):
result[0] = r
t0 = time.time()
objectRegistry_[0].objectIdToObjectDefinition.update({
int(k): TypeDescription.deserialize(v)
for k, v in objectIdToObjectDefinition.iteritems()
})
logging.info("Updated object registry in %s seconds.", time.time() - t0)
t0 = time.time()
try:
converter_[0].convert(objectId, objectRegistry_[0], onConverted)
except Exception as e:
logging.error("Converter raised an exception: %s", traceback.format_exc())
raise Exceptions.InternalError("Unable to convert objectId %s" % objectId)
logging.info("Converted to fora in %s seconds", time.time() - t0)
assert result[0] is not None
if isinstance(result[0], PyforaExceptions.PythonToForaConversionError):
return {'isException': True, 'message': result[0].message, 'trace': result[0].trace}
if isinstance(result[0], Exception):
raise Exceptions.SubscribableWebObjectsException(result[0].message)
objectIdToIvc_[objectId] = result[0]
return {'objectId': objectId}
def defineBuiltinExceptionInstance(self, objectId, typename, argsId):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.BuiltinExceptionInstance(typename, argsId)
def defineRemotePythonObject(self, objectId, computedValueArg):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.RemotePythonObject(computedValueArg)
def defineDict(self, objectId, keyIds, valueIds):
self.objectIdToObjectDefinition[objectId] = TypeDescription.Dict(keyIds=keyIds,
valueIds=valueIds)
def defineFile(self, objectId, text, path):
self.objectIdToObjectDefinition[objectId] = TypeDescription.File(path, text)
def deserialize(self, objectDefinition):
self.assertTrue(hasattr(objectDefinition, 'typeName'))
return TypeDescription.deserialize(objectDefinition)
def deserializeFromStream(stream, objectVisitor, convertJsonToObject):
while True:
objectId = stream.readInt64()
#this is the termination condition
if objectId == -1:
return
code = stream.readByte()
def readSimplePrimitive():
code = stream.readByte()
if code == BinaryObjectRegistry.CODE_NONE:
return None
if code == BinaryObjectRegistry.CODE_INT:
return stream.readInt64()
if code == BinaryObjectRegistry.CODE_STR:
return stream.readString()
if code == BinaryObjectRegistry.CODE_TUPLE:
ct = stream.readInt32()
return tuple([readSimplePrimitive() for _ in xrange(ct)])
if code == BinaryObjectRegistry.CODE_LIST:
ct = stream.readInt32()
return [readSimplePrimitive() for _ in xrange(ct)]
if code == BinaryObjectRegistry.CODE_DICT:
ct = stream.readInt32()
return dict([(readSimplePrimitive(),readSimplePrimitive()) for _ in xrange(ct)])
else:
assert False, "unknown code: " + str(code)
def readInt64s():
return [stream.readInt64() for _ in xrange(stream.readInt64())]
def readStringTuple():
return tuple([stream.readString() for _ in xrange(stream.readInt32())])
def readDottedScopeIds():
res = {}
ct = stream.readInt32()
for _ in xrange(ct):
path = stream.readString()
objId = stream.readInt64()
res[path] = objId
return res
def readPrimitive(code):
if code == BinaryObjectRegistry.CODE_NONE:
return None
elif code == BinaryObjectRegistry.CODE_INT:
return stream.readInt64()
elif code == BinaryObjectRegistry.CODE_LONG:
return long(stream.readString())
elif code == BinaryObjectRegistry.CODE_FLOAT:
return stream.readFloat64()
elif code == BinaryObjectRegistry.CODE_BOOL:
return True if stream.readByte() else False
elif code == BinaryObjectRegistry.CODE_STR:
return stream.readString()
elif code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES:
return [readPrimitive(stream.readByte()) for _ in xrange(stream.readInt64())]
else:
assert False, "unknown code: " + str(code)
if (code == BinaryObjectRegistry.CODE_NONE or
code == BinaryObjectRegistry.CODE_INT or
code == BinaryObjectRegistry.CODE_LONG or
code == BinaryObjectRegistry.CODE_FLOAT or
code == BinaryObjectRegistry.CODE_BOOL or
code == BinaryObjectRegistry.CODE_STR or
code == BinaryObjectRegistry.CODE_LIST_OF_PRIMITIVES):
objectVisitor.definePrimitive(objectId, readPrimitive(code))
elif code == BinaryObjectRegistry.CODE_TUPLE:
objectVisitor.defineTuple(objectId, readInt64s())
elif code == BinaryObjectRegistry.CODE_PACKED_HOMOGENOUS_DATA:
dtype = readSimplePrimitive()
packedBytes = stream.readString()
objectVisitor.definePackedHomogenousData(objectId, TypeDescription.PackedHomogenousData(dtype, packedBytes))
elif code == BinaryObjectRegistry.CODE_LIST:
objectVisitor.defineList(objectId, readInt64s())
elif code == BinaryObjectRegistry.CODE_FILE:
path = stream.readString()
text = stream.readString()
objectVisitor.defineFile(objectId, text, path)
elif code == BinaryObjectRegistry.CODE_DICT:
objectVisitor.defineDict(objectId, readInt64s(), readInt64s())
elif code == BinaryObjectRegistry.CODE_REMOTE_PY_OBJECT:
jsonRepresentation = json.loads(stream.readString())
objectVisitor.defineRemotePythonObject(objectId, convertJsonToObject(jsonRepresentation))
elif code == BinaryObjectRegistry.CODE_BUILTIN_EXCEPTION_INSTANCE:
objectVisitor.defineBuiltinExceptionInstance(objectId, stream.readString(), stream.readInt64())
elif code == BinaryObjectRegistry.CODE_NAMED_SINGLETON:
objectVisitor.defineNamedSingleton(objectId, stream.readString())
elif code == BinaryObjectRegistry.CODE_FUNCTION:
objectVisitor.defineFunction(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds())
elif code == BinaryObjectRegistry.CODE_CLASS:
objectVisitor.defineClass(objectId, stream.readInt64(), stream.readInt32(), readDottedScopeIds(), readInt64s())
elif code == BinaryObjectRegistry.CODE_UNCONVERTIBLE:
if stream.readByte() != 0:
objectVisitor.defineUnconvertible(objectId, readStringTuple())
else:
objectVisitor.defineUnconvertible(objectId, None)
elif code == BinaryObjectRegistry.CODE_CLASS_INSTANCE:
classId = stream.readInt64()
classMembers = {}
for _ in xrange(stream.readInt32()):
memberName = stream.readString()
classMembers[memberName] = stream.readInt64()
objectVisitor.defineClassInstance(objectId, classId, classMembers)
elif code == BinaryObjectRegistry.CODE_INSTANCE_METHOD:
objectVisitor.defineInstanceMethod(objectId, stream.readInt64(), stream.readString())
elif code == BinaryObjectRegistry.CODE_WITH_BLOCK:
scopes = readDottedScopeIds()
objectVisitor.defineWithBlock(objectId, scopes, stream.readInt64(), stream.readInt32())
elif code == BinaryObjectRegistry.CODE_PY_ABORT_EXCEPTION:
typename = stream.readString()
argsId = stream.readInt64()
objectVisitor.definePyAbortException(objectId, typename, argsId)
elif code == BinaryObjectRegistry.CODE_STACKTRACE_AS_JSON:
stackAsJson = stream.readString()
objectVisitor.defineStacktrace(objectId, json.loads(stackAsJson))
elif code == BinaryObjectRegistry.CODE_UNRESOLVED_SYMBOL:
varname = stream.readString()
lineno = stream.readInt64()
col_offset = stream.readInt64()
objectVisitor.defineUnresolvedVarWithPosition(objectId, varname, lineno, col_offset)
else:
assert False, "BinaryStreamDeserializer.deserializeFromStream: unknown code: " + str(code)
def definePrimitive(self, objectId, primitive):
self.objectIdToObjectDefinition[objectId] = TypeDescription.Primitive(
primitive)
def convertStronglyConnectedComponentWithOneNode(
self, dependencyGraph, stronglyConnectedComponent,
objectIdToObjectDefinition):
objectId = stronglyConnectedComponent[0]
objectDefinition = objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or isinstance(
objectDefinition, list):
self.convertedValues[objectId] = self.convertPrimitive(
objectDefinition)
elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition)):
self.convertStronglyConnectedComponentWithOneFunctionOrClass(
objectId, objectDefinition, objectIdToObjectDefinition)
elif isinstance(objectDefinition,
TypeDescription.ClassInstanceDescription):
self.convertClassInstanceDescription(objectId, objectDefinition)
elif isinstance(objectDefinition, TypeDescription.List):
self.convertedValues[objectId] = self.convertList(
objectId,
self._computeRestrictedGraph(objectId, dependencyGraph),
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.Tuple):
self.convertedValues[objectId] = self.convertTuple(
objectId,
self._computeRestrictedGraph(objectId, dependencyGraph),
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.Dict):
self.convertedValues[objectId] = self.convertDict(
objectId, objectDefinition,
self._computeRestrictedGraph(objectId, dependencyGraph),
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.File):
self.convertedValues[objectId] = self.convertFile(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
self.convertedValues[objectId] = self.convertRemotePythonObject(
objectDefinition)
elif isinstance(objectDefinition,
TypeDescription.BuiltinExceptionInstance):
self.convertedValues[
objectId] = self.convertBuiltinExceptionInstance(
objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
self.convertedValues[objectId] = self.convertNamedSingleton(
objectDefinition)
elif isinstance(objectDefinition,
TypeDescription.WithBlockDescription):
self.convertedValues[objectId] = self.convertWithBlock(
objectId, objectDefinition, objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.InstanceMethod):
self.convertedValues[objectId] = self.convertInstanceMethod(
objectId, objectDefinition, objectIdToObjectDefinition)
else:
assert False, "haven't gotten to this yet %s" % type(
objectDefinition)
def test_nested_type_serialization_and_deserialization(self):
dict1 = { "a": 1, "b": 2, "c": 3}
dict2 = { "a": 4, "b": 5, "c": 6}
typeDescription = TypeDescription.ClassDefinition("source text here", [dict1, dict2])
self.assertDeserializationSerializationSucceeded(typeDescription)
def transformBody(objId, implval):
if isUnconvertibleValueTuple(implval):
path = implval[0].pyval
stream.defineUnconvertible(objId, path)
return objId
if isPyforaNameErrorTuple(implval):
payload = implval[0]
varname = payload[0].pyval
lineno = payload[1].pyval
col_offset = payload[2].pyval
stream.defineUnresolvedVarWithPosition(
objId,
varname,
lineno,
col_offset)
return objId
if implval.isString():
value = (implval.pyval,)
else:
value = self.nativeConverterAdaptor.invertForaConstant(implval)
if value is not None:
if isinstance(value, tuple):
#this is a simple constant
stream.definePrimitive(objId, value[0])
return objId
else:
#this is a vector that represents a string
assert isinstance(value, ForaNative.ImplValContainer)
if len(value) == 0:
stream.definePrimitive(objId, "")
return objId
assert value.isVectorOfChar(), value
contents = vectorContentsExtractor(value)
if contents is None:
anyNeedLoading[0] = True
stream.definePrimitive(objId, "")
return objId
else:
assert 'string' in contents
stream.definePrimitive(objId, contents['string'])
return objId
if self.singletonAndExceptionConverter is not None:
value = self.singletonAndExceptionConverter.convertInstanceToSingletonName(implval)
if value is not None:
stream.defineNamedSingleton(objId, value)
return objId
value = self.singletonAndExceptionConverter.convertExceptionInstance(implval)
if value is not None:
stream.defineBuiltinExceptionInstance(objId, value[0], transform(value[1]))
return objId
value = self.singletonAndExceptionConverter.convertPyAbortExceptionInstance(
implval
)
if value is not None:
stream.definePyAbortException(objId, value[0], transform(value[1]))
return objId
value = self.nativeConverterAdaptor.invertTuple(implval)
if value is not None:
tupleIds = tuple([transform(x) for x in value])
stream.defineTuple(objId, tupleIds)
return objId
value = self.nativeConverterAdaptor.invertDict(implval)
if value is not None:
stream.defineDict(
objId,
[transform(k) for k in value.keys()],
[transform(v) for v in value.values()]
)
return objId
listItemsAsVector = self.nativeConverterAdaptor.invertList(implval)
if listItemsAsVector is not None:
contents = vectorContentsExtractor(listItemsAsVector)
if contents is None:
stream.defineList(objId, [])
anyNeedLoading[0] = True
return objId
elif 'listContents' in contents:
stream.defineList(objId, [transform(x) for x in contents['listContents']])
return objId
else:
assert 'contentsAsNumpyArray' in contents
contentsAsNumpy = contents['contentsAsNumpyArray']
stream.definePackedHomogenousData(
objId,
TypeDescription.PackedHomogenousData(
TypeDescription.dtypeToPrimitive(contentsAsNumpy.dtype),
contentsAsNumpy.tostring()
)
)
return objId
if implval.isStackTrace():
stackTraceAsJsonOrNone = self.getStackTraceAsJsonOrNone(implval)
if stackTraceAsJsonOrNone is not None:
stream.defineStacktrace(objId, stackTraceAsJsonOrNone)
return objId
else:
assert False, "unknown tuple, but not a stacktrace: %s" % implval
if implval.isObject():
objectClass = implval.getObjectClass()
if objectClass == self.pyforaBoundMethodClass:
nameAsImplval = implval.getObjectLexicalMember("@name")[0]
if not nameAsImplval.isSymbol():
raise pyfora.ForaToPythonConversionError(
"PyBoundMethod found with name %s of type %s, which should be a symbol but is not."
% (nameAsImplval, nameAsImplval.type)
)
stream.defineInstanceMethod(
objId,
transform(implval.getObjectLexicalMember("@self")[0]),
nameAsImplval.pyval[1:]
)
return objId
sourcePathAndLine = self.extractSourcePathAndLine(implval)
if sourcePathAndLine is not None:
if objectClass is not None:
classObjectId = transform(objectClass)
members = {}
for memberName in objectClass.objectMembers:
if memberName is not None:
member = implval.getObjectLexicalMember(memberName)
if member is not None and member[1] is None:
assert memberName == "@m"
assert member[0].isTuple()
membersTuple = member[0]
memberNames = membersTuple.getTupleNames()
for i, name in enumerate(memberNames):
result = transform(membersTuple[i])
if result is not UnconvertibleToken:
members[str(name)] = transform(membersTuple[i])
stream.defineClassInstance(objId, classObjectId, members)
return objId
else:
members = {}
lexicalMembers = implval.objectLexicalMembers
for memberAndBindingSequence in lexicalMembers.iteritems():
#if the binding sequence is empty, then this binding refers to 'self'
if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]:
memberName = memberAndBindingSequence[0]
member = implval.getObjectLexicalMember(memberName)
if member is not None and member[1] is None:
if member[0] != Symbol_uninitialized:
transformed = transform(member[0])
if transformed is not UnconvertibleToken:
members[(str(memberName),)] = transformed
sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0])
stream.defineFunction(
objId,
sourceFileId,
sourcePathAndLine[2],
members
)
return objId
elif implval.isClass():
members = {}
sourcePathAndLine = self.extractSourcePathAndLine(implval)
lexicalMembers = implval.objectLexicalMembers
for memberAndBindingSequence in lexicalMembers.iteritems():
#if the binding sequence is empty, then this binding refers to 'self'
if isinstance(memberAndBindingSequence[1], ForaNative.ImplValContainer) or memberAndBindingSequence[1][0]:
memberName = memberAndBindingSequence[0]
member = implval.getObjectLexicalMember(memberName)
if member is not None and member[1] is None:
result = transform(member[0])
if result is not UnconvertibleToken:
members[(str(memberName),)] = result
om = implval.objectMetadata
if 'classMetadata' in om:
om = om['classMetadata']
om = om['sourceText'].objectMetadata
sourceFileId = transformFile(sourcePathAndLine[1], sourcePathAndLine[0])
stream.defineClass(
objId,
sourceFileId,
sourcePathAndLine[2],
members,
#this is a little wrong. When going python->binary, we would have
#a list of base classes. When going pyfora->binary, we're holding the
#base classes by name as free variables.
()
)
return objId
logging.error("Failed to convert %s of type %s back to python", implval, str(implval.type))
raise pyfora.ForaToPythonConversionError(
"Result cannot be translated back to python."
)
def convertStronglyConnectedComponentWithOneNode(self,
dependencyGraph,
stronglyConnectedComponent,
objectIdToObjectDefinition):
objectId = stronglyConnectedComponent[0]
objectDefinition = objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list):
self.convertedValues[objectId] = self.convertPrimitive(objectDefinition)
elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition)):
if isinstance(objectDefinition, TypeDescription.ClassDefinition):
for baseId in objectDefinition.baseClassIds:
if baseId not in self.convertedValues:
self._convert(baseId,
dependencyGraph,
objectIdToObjectDefinition)
assert baseId in self.convertedValues
self.convertStronglyConnectedComponentWithOneFunctionOrClass(
objectId,
objectDefinition,
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription):
self.convertClassInstanceDescription(
objectId,
objectDefinition
)
elif isinstance(objectDefinition, TypeDescription.List):
self.convertedValues[objectId] = self.convertList(
objectId,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.Tuple):
self.convertedValues[objectId] = self.convertTuple(
objectId,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.Dict):
self.convertedValues[objectId] = self.convertDict(
objectId,
objectDefinition,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.File):
self.convertedValues[objectId] = self.convertFile(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
self.convertedValues[objectId] = self.convertRemotePythonObject(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance):
self.convertedValues[objectId] = self.convertBuiltinExceptionInstance(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
self.convertedValues[objectId] = self.convertNamedSingleton(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.WithBlockDescription):
self.convertWithBlock(objectId,
objectDefinition,
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.InstanceMethod):
self.convertedValues[objectId] = self.convertInstanceMethod(objectId,
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.Unconvertible):
self.convertUnconvertibleValue(objectId, objectDefinition.module_path)
elif isinstance(objectDefinition, TypeDescription.PackedHomogenousData):
self.convertedValues[objectId] = self.convertPackedHomogenousDataAsList(
objectId,
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.UnresolvedVarWithPosition):
self.convertedValues[objectId] = self.convertUnresolvedVarWithPosition(
objectId,
objectDefinition
)
else:
assert False, "haven't gotten to this yet %s" % type(objectDefinition)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition):
objectDefinition = objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list):
return self.convertPrimitive(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
return self.convertRemotePythonObject(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
return self.convertNamedSingleton(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance):
return self.convertBuiltinExceptionInstance(
objectDefinition
)
elif isinstance(objectDefinition,
(TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition,
TypeDescription.ClassInstanceDescription,
TypeDescription.InstanceMethod)
):
return (
self.convertObjectWithDependencies(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.List):
return (
self.convertList(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.PackedHomogenousData):
return (
self.convertPackedHomogenousDataAsList(
objectId,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Tuple):
return (
self.convertTuple(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Dict):
return (
self.convertDict(
objectId,
objectDefinition,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.WithBlockDescription):
return (
self.convertObjectWithDependencies(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Unconvertible):
return self.convertUnconvertibleValue(objectId, objectDefinition.module_path)
elif isinstance(objectDefinition, TypeDescription.UnresolvedVarWithPosition):
return self.convertUnresolvedVarWithPosition(objectId, objectDefinition)
else:
raise pyfora.PythonToForaConversionError(
"don't know how to convert %s of type %s" % (
objectDefinition, type(objectDefinition)
)
)
def defineNamedSingleton(self, objectId, singletonName):
self.objectIdToObjectDefinition[objectId] = TypeDescription.NamedSingleton(singletonName)
def _convert(self, objectId, dependencyGraph, objectIdToObjectDefinition):
objectDefinition = objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list):
return self.convertPrimitive(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
return self.convertRemotePythonObject(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
return self.convertNamedSingleton(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance):
return self.convertBuiltinExceptionInstance(
objectDefinition
)
elif isinstance(objectDefinition,
(TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition,
TypeDescription.ClassInstanceDescription,
TypeDescription.InstanceMethod)
):
return (
self.convertObjectWithDependencies(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.List):
return (
self.convertList(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Tuple):
return (
self.convertTuple(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Dict):
return (
self.convertDict(
objectId,
objectDefinition,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.WithBlockDescription):
return (
self.convertObjectWithDependencies(
objectId,
dependencyGraph,
objectIdToObjectDefinition
)
)
elif isinstance(objectDefinition, TypeDescription.Unconvertible):
return self.convertUnconvertibleValue(objectId)
else:
raise pyfora.PythonToForaConversionError(
"don't know how to convert %s of type %s" % (
objectDefinition, type(objectDefinition)
)
)
def defineTuple(self, objectId, memberIds):
self.objectIdToObjectDefinition[objectId] = TypeDescription.Tuple(memberIds)
def defineUnconvertible(self, objectId):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.Unconvertible()
def defineList(self, objectId, memberIds):
self.objectIdToObjectDefinition[objectId] = TypeDescription.List(memberIds)
def defineInstanceMethod(self, objectId, instanceId, methodName):
self.objectIdToObjectDefinition[objectId] = \
TypeDescription.InstanceMethod(
instanceId=instanceId,
methodName=methodName
)
def assertDeserializationSerializationSucceeded(self, typeDescription):
jsonString = json.dumps(typeDescription)
jobject = json.loads(jsonString)
deserializationResult = TypeDescription.fromList(jobject)
self.assertNamedTuplesAreTheSame(deserializationResult, typeDescription)
def definePyAbortException(self, objectId, typename, argsId):
self.objectIdToObjectDefinition[
objectId] = TypeDescription.PyAbortException(typename, argsId)
def convertStronglyConnectedComponentWithOneNode(self,
dependencyGraph,
stronglyConnectedComponent,
objectIdToObjectDefinition):
objectId = stronglyConnectedComponent[0]
objectDefinition = objectIdToObjectDefinition[objectId]
if TypeDescription.isPrimitive(objectDefinition) or isinstance(objectDefinition, list):
self.convertedValues[objectId] = self.convertPrimitive(objectDefinition)
elif isinstance(objectDefinition, (TypeDescription.FunctionDefinition,
TypeDescription.ClassDefinition)):
if isinstance(objectDefinition, TypeDescription.ClassDefinition):
for baseId in objectDefinition.baseClassIds:
if baseId not in self.convertedValues:
self._convert(baseId,
dependencyGraph,
objectIdToObjectDefinition)
assert baseId in self.convertedValues
self.convertStronglyConnectedComponentWithOneFunctionOrClass(
objectId,
objectDefinition,
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.ClassInstanceDescription):
self.convertClassInstanceDescription(
objectId,
objectDefinition
)
elif isinstance(objectDefinition, TypeDescription.List):
self.convertedValues[objectId] = self.convertList(
objectId,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.Tuple):
self.convertedValues[objectId] = self.convertTuple(
objectId,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.Dict):
self.convertedValues[objectId] = self.convertDict(
objectId,
objectDefinition,
self._computeRestrictedGraph(
objectId,
dependencyGraph
),
objectIdToObjectDefinition
)
elif isinstance(objectDefinition, TypeDescription.File):
self.convertedValues[objectId] = self.convertFile(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.RemotePythonObject):
self.convertedValues[objectId] = self.convertRemotePythonObject(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.BuiltinExceptionInstance):
self.convertedValues[objectId] = self.convertBuiltinExceptionInstance(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.NamedSingleton):
self.convertedValues[objectId] = self.convertNamedSingleton(objectDefinition)
elif isinstance(objectDefinition, TypeDescription.WithBlockDescription):
self.convertWithBlock(objectId,
objectDefinition,
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.InstanceMethod):
self.convertedValues[objectId] = self.convertInstanceMethod(objectId,
objectIdToObjectDefinition)
elif isinstance(objectDefinition, TypeDescription.Unconvertible):
self.convertedValues[objectId] = Symbol_unconvertible
else:
assert False, "haven't gotten to this yet %s" % type(objectDefinition)
