def toType(pat):
"""Convert a pattern to a type, if possible (wildcards can't be converted to types).
:type pat: titus.P
:param pat: pattern to convert
:rtype: titus.datatype.Type
:return: corresponding type (titus.datatype.Type rather than titus.datatype.AvroType to allow for titus.datatype.FcnType)
"""
if isinstance(pat, Null): return AvroNull()
elif isinstance(pat, Boolean): return AvroBoolean()
elif isinstance(pat, Int): return AvroInt()
elif isinstance(pat, Long): return AvroLong()
elif isinstance(pat, Float): return AvroFloat()
elif isinstance(pat, Double): return AvroDouble()
elif isinstance(pat, Bytes): return AvroBytes()
elif isinstance(pat, String): return AvroString()
elif isinstance(pat, Array): return AvroArray(toType(pat.items))
elif isinstance(pat, Map): return AvroMap(toType(pat.values))
elif isinstance(pat, Union):
return AvroUnion([toType(x) for x in pat.types])
elif isinstance(pat, Fixed) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroFixed(pat.size, namebits[-1], None)
else:
return AvroFixed(pat.size, namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Fixed):
return AvroFixed(pat.size)
elif isinstance(pat, Enum) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroEnum(pat.symbols, namebits[-1], None)
else:
return AvroEnum(pat.symbols, namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Enum):
return AvroEnum(pat.symbols)
elif isinstance(pat, Record) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroRecord(
[AvroField(k, toType(v)) for k, v in list(pat.fields.items())],
namebits[-1], None)
else:
return AvroRecord(
[AvroField(k, toType(v)) for k, v in list(pat.fields.items())],
namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Record):
return AvroRecord(
[AvroField(k, toType(v)) for k, v in list(pat.fields.items())])
elif isinstance(pat, Fcn):
return FcnType([toType(x) for x in pat.params()], toType(pat.ret()))
else:
raise Exception
def testKMeansTransform(self):
random.seed(12345)
numpy.seterr(divide="ignore", invalid="ignore")
dataset = numpy.empty((100000, 3), dtype=numpy.dtype(float))
for i, (x, y, z) in enumerate(
TestProducerKMeans.data([2, 1, 1], [3, 2, 5], [8, 2, 7],
[5, 8, 5], [1, 1, 9])):
if i >= dataset.shape[0]:
break
dataset[i, :] = [x * 10.0, y * 20.0, z * 30.0]
trans = Transformation("x/10.0", "y/20.0", "z/30.0")
kmeans = KMeans(5, trans.transform(dataset, ["x", "y", "z"]))
kmeans.optimize(whileall(moving(), maxIterations(1000)))
centers = kmeans.centers()
self.assertArrayAlmostEqual(centers[0], [1.01, 1.00, 9.01], places=1)
self.assertArrayAlmostEqual(centers[1], [2.00, 1.01, 1.00], places=1)
self.assertArrayAlmostEqual(centers[2], [3.01, 2.01, 5.00], places=1)
self.assertArrayAlmostEqual(centers[3], [4.99, 8.00, 4.99], places=1)
self.assertArrayAlmostEqual(centers[4], [8.02, 2.00, 7.01], places=1)
doc = kmeans.pfaDocument("Cluster",
["one", "two", "three", "four", "five"],
preprocess=trans.new(AvroArray(AvroDouble()),
x="input[0]",
y="input[1]",
z="input[2]"))
# look(doc, maxDepth=10)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][0]["center"], [1.01, 1.00, 9.01],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][1]["center"], [2.00, 1.01, 1.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][2]["center"], [3.01, 2.01, 5.00],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][3]["center"], [4.99, 8.00, 4.99],
places=1)
self.assertArrayAlmostEqual(
doc["cells"]["clusters"]["init"][4]["center"], [8.02, 2.00, 7.01],
places=1)
engine, = PFAEngine.fromJson(doc)
self.assertEqual(engine.action([1.01 * 10, 1.00 * 20, 9.01 * 30]),
"one")
self.assertEqual(engine.action([2.00 * 10, 1.01 * 20, 1.00 * 30]),
"two")
self.assertEqual(engine.action([3.01 * 10, 2.01 * 20, 5.00 * 30]),
"three")
self.assertEqual(engine.action([4.99 * 10, 8.00 * 20, 4.99 * 30]),
"four")
self.assertEqual(engine.action([8.02 * 10, 2.00 * 20, 7.01 * 30]),
"five")
def assignRet(self, pat, assignments):
"""Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string" etc.) to the return pattern.
:type pat: titus.P
:param pat: original return pattern
:type assignments: dict from label letters to titus.datatype.AvroType
:param assignments: assigned types to apply
:rtype: titus.datatype.AvroType
:return: resolved type for the return value of the signature
"""
if isinstance(pat, P.Null):
return AvroNull()
elif isinstance(pat, P.Boolean):
return AvroBoolean()
elif isinstance(pat, P.Int):
return AvroInt()
elif isinstance(pat, P.Long):
return AvroLong()
elif isinstance(pat, P.Float):
return AvroFloat()
elif isinstance(pat, P.Double):
return AvroDouble()
elif isinstance(pat, P.Bytes):
return AvroBytes()
elif isinstance(pat, P.String):
return AvroString()
elif isinstance(pat, P.Array):
return AvroArray(self.assignRet(pat.items, assignments))
elif isinstance(pat, P.Map):
return AvroMap(self.assignRet(pat.values, assignments))
elif isinstance(pat, P.Union):
return AvroUnion(
[self.assignRet(x, assignments) for x in pat.types])
elif isinstance(pat, P.Fixed):
return P.toType(pat)
elif isinstance(pat, P.Enum):
return P.toType(pat)
elif isinstance(pat, P.Record):
return P.toType(pat)
elif isinstance(pat, P.Fcn):
return P.toType(pat)
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception(repr(pat))
def assign(self, pat, arg, assignments):
"""Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string", etc.) to each parameter of the signature.
:type pat: titus.P
:param pat: original parameter pattern
:type arg: titus.datatype.AvroType
:param arg: supplied argument type
:type assignments: dict from label letters to titus.datatype.AvroType
:param assignments: assigned types to apply
:rtype: titus.datatype.AvroType
:return: resolved type for one parameter of the signature
"""
if isinstance(pat, P.Null) and isinstance(arg, AvroNull):
return arg
elif isinstance(pat, P.Boolean) and isinstance(arg, AvroBoolean):
return arg
elif isinstance(pat, P.Int) and isinstance(arg, AvroInt):
return AvroInt()
elif isinstance(pat, P.Long) and (isinstance(arg, AvroInt)
or isinstance(arg, AvroLong)):
return AvroLong()
elif isinstance(pat, P.Float) and (isinstance(arg, AvroInt)
or isinstance(arg, AvroLong)
or isinstance(arg, AvroFloat)):
return AvroFloat()
elif isinstance(pat, P.Double) and (isinstance(arg, AvroInt)
or isinstance(arg, AvroLong)
or isinstance(arg, AvroFloat)
or isinstance(arg, AvroDouble)):
return AvroDouble()
elif isinstance(pat, P.Bytes) and isinstance(arg, AvroBytes):
return arg
elif isinstance(pat, P.String) and isinstance(arg, AvroString):
return arg
elif isinstance(pat, P.Array) and isinstance(arg, AvroArray):
return AvroArray(
P.mustBeAvro(self.assign(pat.items, arg.items, assignments)))
elif isinstance(pat, P.Map) and isinstance(arg, AvroMap):
return AvroMap(
P.mustBeAvro(self.assign(pat.values, arg.values, assignments)))
elif isinstance(pat, P.Union) and isinstance(arg, AvroUnion):
return arg
elif isinstance(pat, P.Union) and isinstance(arg, AvroType):
return arg
elif isinstance(pat, P.Fixed) and isinstance(arg, AvroFixed):
return arg
elif isinstance(pat, P.Enum) and isinstance(arg, AvroEnum):
return arg
elif isinstance(pat, P.Record) and isinstance(arg, AvroRecord):
return arg
elif isinstance(pat, P.Fcn) and isinstance(arg, FcnType):
return arg
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception
def broadestType(candidates):
"""Compute the narrowest possible supertype of a set of types.
:type candidates: list of titus.datatype.AvroType
:param candidates: set of types for which to find the narrowest possible supertype
:rtype: titus.datatype.AvroType
:return: narrowest possible supertype, usually a union of the candidates
"""
realCandidates = [
x for x in candidates if not isinstance(x, ExceptionType)
]
if len(candidates) == 0:
return ValueError("empty list of types")
elif len(realCandidates) == 0:
return ValueError("list of types consists only of exception type")
elif all(isinstance(x, AvroNull) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroBoolean) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroInt) for x in realCandidates):
return AvroInt()
elif all(
isinstance(x, AvroInt) or isinstance(x, AvroLong)
for x in realCandidates):
return AvroLong()
elif all(
isinstance(x, AvroInt) or isinstance(x, AvroLong)
or isinstance(x, AvroFloat) for x in realCandidates):
return AvroFloat()
elif all(
isinstance(x, AvroInt) or isinstance(x, AvroLong)
or isinstance(x, AvroFloat) or isinstance(x, AvroDouble)
for x in realCandidates):
return AvroDouble()
elif all(isinstance(x, AvroBytes) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroString) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroArray) for x in realCandidates):
return AvroArray(
P.mustBeAvro(
LabelData.broadestType([x.items for x in realCandidates])))
elif all(isinstance(x, AvroMap) for x in realCandidates):
return AvroMap(
P.mustBeAvro(
LabelData.broadestType([x.values
for x in realCandidates])))
elif all(isinstance(x, AvroFixed) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible fixed types: " +
" ".join(map(repr, realCandidates)))
elif all(isinstance(x, AvroEnum) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible enum types: " +
" ".join(map(repr, realCandidates)))
elif all(isinstance(x, AvroRecord) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible record types: " +
" ".join(map(repr, realCandidates)))
elif all(isinstance(x, FcnType) for x in realCandidates):
params = realCandidates[0].params
ret = realCandidates[0].ret
if all(x.params == params and x.ret == ret
for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible function types: " +
" ".join(map(repr, realCandidates)))
elif not any(isinstance(x, FcnType) for x in realCandidates):
types = LabelData.distinctTypes(realCandidates)
types = [P.mustBeAvro(x) for x in types]
countFixed = 0
countEnum = 0
for t in types:
if isinstance(t, AvroFixed):
countFixed += 1
if isinstance(t, AvroEnum):
countEnum += 1
if countFixed > 1:
raise IncompatibleTypes("incompatible fixed types")
if countEnum > 1:
raise IncompatibleTypes("incompatible enum types")
return AvroUnion(types)
else:
raise IncompatibleTypes(
"incompatible function/non-function types: " +
" ".join(map(repr, realCandidates)))