def assignRet(self, pat, assignments):
"""Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string" etc.) to the return pattern.
:type pat: titus.P
:param pat: original return pattern
:type assignments: dict from label letters to titus.datatype.AvroType
:param assignments: assigned types to apply
:rtype: titus.datatype.AvroType
:return: resolved type for the return value of the signature
"""
if isinstance(pat, P.Null):
return AvroNull()
elif isinstance(pat, P.Boolean):
return AvroBoolean()
elif isinstance(pat, P.Int):
return AvroInt()
elif isinstance(pat, P.Long):
return AvroLong()
elif isinstance(pat, P.Float):
return AvroFloat()
elif isinstance(pat, P.Double):
return AvroDouble()
elif isinstance(pat, P.Bytes):
return AvroBytes()
elif isinstance(pat, P.String):
return AvroString()
elif isinstance(pat, P.Array):
return AvroArray(self.assignRet(pat.items, assignments))
elif isinstance(pat, P.Map):
return AvroMap(self.assignRet(pat.values, assignments))
elif isinstance(pat, P.Union):
return AvroUnion([self.assignRet(x, assignments) for x in pat.types])
elif isinstance(pat, P.Fixed):
return P.toType(pat)
elif isinstance(pat, P.Enum):
return P.toType(pat)
elif isinstance(pat, P.Record):
return P.toType(pat)
elif isinstance(pat, P.Fcn):
return P.toType(pat)
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception(repr(pat))
def assign(self, pat, arg, assignments):
if isinstance(pat, P.Null) and isinstance(arg, AvroNull):
return arg
elif isinstance(pat, P.Boolean) and isinstance(arg, AvroBoolean):
return arg
elif isinstance(pat, P.Int) and isinstance(arg, AvroInt):
return AvroInt()
elif isinstance(pat, P.Long) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong)):
return AvroLong()
elif isinstance(pat, P.Float) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat)):
return AvroFloat()
elif isinstance(pat, P.Double) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat) or isinstance(arg, AvroDouble)):
return AvroDouble()
elif isinstance(pat, P.Bytes) and isinstance(arg, AvroBytes):
return arg
elif isinstance(pat, P.String) and isinstance(arg, AvroString):
return arg
elif isinstance(pat, P.Array) and isinstance(arg, AvroArray):
return AvroArray(P.mustBeAvro(self.assign(pat.items, arg.items, assignments)))
elif isinstance(pat, P.Map) and isinstance(arg, AvroMap):
return AvroMap(P.mustBeAvro(self.assign(pat.values, arg.values, assignments)))
elif isinstance(pat, P.Union) and isinstance(arg, AvroUnion):
return arg
elif isinstance(pat, P.Union) and isinstance(arg, AvroType):
return arg
elif isinstance(pat, P.Fixed) and isinstance(arg, AvroFixed):
return arg
elif isinstance(pat, P.Enum) and isinstance(arg, AvroEnum):
return arg
elif isinstance(pat, P.Record) and isinstance(arg, AvroRecord):
return arg
elif isinstance(pat, P.Fcn) and isinstance(arg, FcnType):
return arg
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception
def assignRet(self, pat, assignments):
if isinstance(pat, P.Null):
return AvroNull()
elif isinstance(pat, P.Boolean):
return AvroBoolean()
elif isinstance(pat, P.Int):
return AvroInt()
elif isinstance(pat, P.Long):
return AvroLong()
elif isinstance(pat, P.Float):
return AvroFloat()
elif isinstance(pat, P.Double):
return AvroDouble()
elif isinstance(pat, P.Bytes):
return AvroBytes()
elif isinstance(pat, P.String):
return AvroString()
elif isinstance(pat, P.Array):
return AvroArray(self.assignRet(pat.items, assignments))
elif isinstance(pat, P.Map):
return AvroMap(self.assignRet(pat.values, assignments))
elif isinstance(pat, P.Union):
return AvroUnion([self.assignRet(x, assignments) for x in pat.types])
elif isinstance(pat, P.Fixed):
return P.toType(pat)
elif isinstance(pat, P.Enum):
return P.toType(pat)
elif isinstance(pat, P.Record):
return P.toType(pat)
elif isinstance(pat, P.Fcn):
return P.toType(pat)
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception(repr(pat))
def assign(self, pat, arg, assignments):
"""Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string", etc.) to each parameter of the signature.
:type pat: titus.P
:param pat: original parameter pattern
:type arg: titus.datatype.AvroType
:param arg: supplied argument type
:type assignments: dict from label letters to titus.datatype.AvroType
:param assignments: assigned types to apply
:rtype: titus.datatype.AvroType
:return: resolved type for one parameter of the signature
"""
if isinstance(pat, P.Null) and isinstance(arg, AvroNull):
return arg
elif isinstance(pat, P.Boolean) and isinstance(arg, AvroBoolean):
return arg
elif isinstance(pat, P.Int) and isinstance(arg, AvroInt):
return AvroInt()
elif isinstance(pat, P.Long) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong)):
return AvroLong()
elif isinstance(pat, P.Float) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat)):
return AvroFloat()
elif isinstance(pat, P.Double) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat) or isinstance(arg, AvroDouble)):
return AvroDouble()
elif isinstance(pat, P.Bytes) and isinstance(arg, AvroBytes):
return arg
elif isinstance(pat, P.String) and isinstance(arg, AvroString):
return arg
elif isinstance(pat, P.Array) and isinstance(arg, AvroArray):
return AvroArray(P.mustBeAvro(self.assign(pat.items, arg.items, assignments)))
elif isinstance(pat, P.Map) and isinstance(arg, AvroMap):
return AvroMap(P.mustBeAvro(self.assign(pat.values, arg.values, assignments)))
elif isinstance(pat, P.Union) and isinstance(arg, AvroUnion):
return arg
elif isinstance(pat, P.Union) and isinstance(arg, AvroType):
return arg
elif isinstance(pat, P.Fixed) and isinstance(arg, AvroFixed):
return arg
elif isinstance(pat, P.Enum) and isinstance(arg, AvroEnum):
return arg
elif isinstance(pat, P.Record) and isinstance(arg, AvroRecord):
return arg
elif isinstance(pat, P.Fcn) and isinstance(arg, FcnType):
return arg
elif isinstance(pat, P.Wildcard):
return assignments[pat.label]
elif isinstance(pat, P.WildRecord):
return assignments[pat.label]
elif isinstance(pat, P.WildEnum):
return assignments[pat.label]
elif isinstance(pat, P.WildFixed):
return assignments[pat.label]
elif isinstance(pat, P.EnumFields):
return assignments[pat.label]
else:
raise Exception
def broadestType(candidates):
"""Compute the narrowest possible supertype of a set of types.
:type candidates: list of titus.datatype.AvroType
:param candidates: set of types for which to find the narrowest possible supertype
:rtype: titus.datatype.AvroType
:return: narrowest possible supertype, usually a union of the candidates
"""
realCandidates = [x for x in candidates if not isinstance(x, ExceptionType)]
if len(candidates) == 0:
return ValueError("empty list of types")
elif len(realCandidates) == 0:
return ValueError("list of types consists only of exception type")
elif all(isinstance(x, AvroNull) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroBoolean) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroInt) for x in realCandidates):
return AvroInt()
elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) for x in realCandidates):
return AvroLong()
elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) or isinstance(x, AvroFloat) for x in realCandidates):
return AvroFloat()
elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) or isinstance(x, AvroFloat) or isinstance(x, AvroDouble) for x in realCandidates):
return AvroDouble()
elif all(isinstance(x, AvroBytes) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroString) for x in realCandidates):
return realCandidates[0]
elif all(isinstance(x, AvroArray) for x in realCandidates):
return AvroArray(P.mustBeAvro(LabelData.broadestType([x.items for x in realCandidates])))
elif all(isinstance(x, AvroMap) for x in realCandidates):
return AvroMap(P.mustBeAvro(LabelData.broadestType([x.values for x in realCandidates])))
elif all(isinstance(x, AvroFixed) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible fixed types: " + " ".join(map(repr, realCandidates)))
elif all(isinstance(x, AvroEnum) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible enum types: " + " ".join(map(repr, realCandidates)))
elif all(isinstance(x, AvroRecord) for x in realCandidates):
fullName = realCandidates[0].fullName
if all(x.fullName == fullName for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible record types: " + " ".join(map(repr, realCandidates)))
elif all(isinstance(x, FcnType) for x in realCandidates):
params = realCandidates[0].params
ret = realCandidates[0].ret
if all(x.params == params and x.ret == ret for x in realCandidates[1:]):
return realCandidates[0]
else:
raise IncompatibleTypes("incompatible function types: " + " ".join(map(repr, realCandidates)))
elif not any(isinstance(x, FcnType) for x in realCandidates):
types = LabelData.distinctTypes(realCandidates)
types = [P.mustBeAvro(x) for x in types]
countFixed = 0
countEnum = 0
for t in types:
if isinstance(t, AvroFixed):
countFixed += 1
if isinstance(t, AvroEnum):
countEnum += 1
if countFixed > 1:
raise IncompatibleTypes("incompatible fixed types")
if countEnum > 1:
raise IncompatibleTypes("incompatible enum types")
return AvroUnion(types)
else:
raise IncompatibleTypes("incompatible function/non-function types: " + " ".join(map(repr, realCandidates)))
class Bernoulli(LibFcn):
name = prefix + "bernoulli"
sig = Sigs([
Sig([{
"datum": P.Array(P.String())
}, {
"classModel": P.Map(P.Double())
}], P.Double()),
Sig([{
"datum": P.Array(P.String())
}, {
"classModel": P.WildRecord("C", {"values": P.Map(P.Double())})
}], P.Double())
])
errcodeBase = 10020
def __call__(self, state, scope, pos, paramTypes, datum, classModel):
if paramTypes[1]["type"] == "record":
classModel = classModel["values"]
ll = 0.0
for v in classModel.values():
if (v <= 0.0) or (v >= 1.0):
raise PFARuntimeException(
"probability in classModel cannot be less than 0 or greater than 1",
self.errcodeBase + 0, self.name, pos)
ll += math.log(1.0 - v)
for item in datum:
p = classModel.get(item, None)
if p is not None:
ll += math.log(p) - math.log(1.0 - p)
return ll
class FillHistogram2d(LibFcn):
name = prefix + "fillHistogram2d"
sig = Sig([{
"x": P.Double()
}, {
"y": P.Double()
}, {
"w": P.Double()
}, {
"histogram":
P.WildRecord(
"A", {
"xnumbins": P.Int(),
"xlow": P.Double(),
"xhigh": P.Double(),
"ynumbins": P.Int(),
"ylow": P.Double(),
"yhigh": P.Double(),
"values": P.Array(P.Array(P.Double()))
})
}], P.Wildcard("A"))
errcodeBase = 14090
def genpy(self, paramTypes, args, pos):
def has(name):
for x in paramTypes[3].fields:
if x.name == name:
if not x.avroType.accepts(AvroDouble()):
raise PFASemanticException(
"{0} is being given a record type in which the \"{1}\" field is not a double: {2}"
.format(self.name, name, x.avroType), None)
return True
return False
return "self.f[{0}]({1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11})".format(
repr(self.name), ", ".join(
["state", "scope",
repr(pos), repr(paramTypes)] + args), has("underunderflow"),
has("undermidflow"), has("underoverflow"), has("midunderflow"),
has("midoverflow"), has("overunderflow"), has("overmidflow"),
has("overoverflow"), has("nanflow"), has("infflow"))
def __call__(self, state, scope, pos, paramTypes, x, y, w, histogram,
hasUnderunderflow, hasUndermidflow, hasUnderoverflow,
hasMidunderflow, hasMidoverflow, hasOverunderflow,
hasOvermidflow, hasOveroverflow, hasNanflow, hasInfflow):
values = histogram["values"]
xnumbins = histogram["xnumbins"]
xlow = histogram["xlow"]
xhigh = histogram["xhigh"]
ynumbins = histogram["ynumbins"]
ylow = histogram["ylow"]
yhigh = histogram["yhigh"]
if xnumbins < 1 or ynumbins < 1:
raise PFARuntimeException("bad histogram scale",
self.errcodeBase + 2, self.name, pos)
if len(values) != xnumbins or any(len(x) != ynumbins for x in values):
raise PFARuntimeException("wrong histogram size",
self.errcodeBase + 0, self.name, pos)
if xlow >= xhigh or ylow >= yhigh or math.isnan(xlow) or math.isinf(
xlow) or math.isnan(xhigh) or math.isinf(xhigh) or math.isnan(
ylow) or math.isinf(ylow) or math.isnan(
yhigh) or math.isinf(yhigh):
raise PFARuntimeException("bad histogram range",
self.errcodeBase + 1, self.name, pos)
if math.isnan(x) or math.isnan(
y): # do nan check first: nan wins over inf
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, True, False
elif hasInfflow and (math.isinf(x) or math.isinf(y)):
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, False, True
elif x >= xhigh and y >= yhigh:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, True, False, False
elif x >= xhigh and y >= ylow and y < yhigh:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, True, False, False, False
elif x >= xhigh and y < ylow:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, True, False, False, False, False
elif x >= xlow and x < xhigh and y >= yhigh:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, True, False, False, False, False, False
elif x >= xlow and x < xhigh and y < ylow:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, True, False, False, False, False, False, False
elif x < xlow and y >= yhigh:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, True, False, False, False, False, False, False, False
elif x < xlow and y >= ylow and y < yhigh:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, True, False, False, False, False, False, False, False, False
elif x < xlow and y < ylow:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = True, False, False, False, False, False, False, False, False, False
else:
underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, False, False
if not underunderflow and not undermidflow and not underoverflow and not midunderflow and not midoverflow and not overunderflow and not overmidflow and not overoverflow and not nanflow and not infflow:
try:
xindex = int(math.floor(
(x - xlow) / (xhigh - xlow) * xnumbins))
except ZeroDivisionError:
xindex = 0
try:
yindex = int(math.floor(
(y - ylow) / (yhigh - ylow) * ynumbins))
except ZeroDivisionError:
yindex = 0
newValues = [list(x) for x in values]
newValues[xindex][yindex] = newValues[xindex][yindex] + w
else:
newValues = values
updator = {"values": newValues}
if hasUnderunderflow:
updator["underunderflow"] = histogram["underunderflow"] + (
w if underunderflow else 0.0)
if hasUndermidflow:
updator["undermidflow"] = histogram["undermidflow"] + (
w if undermidflow else 0.0)
if hasUnderoverflow:
updator["underoverflow"] = histogram["underoverflow"] + (
w if underoverflow else 0.0)
if hasMidunderflow:
updator["midunderflow"] = histogram["midunderflow"] + (
w if midunderflow else 0.0)
if hasMidoverflow:
updator["midoverflow"] = histogram["midoverflow"] + (
w if midoverflow else 0.0)
if hasOverunderflow:
updator["overunderflow"] = histogram["overunderflow"] + (
w if overunderflow else 0.0)
if hasOvermidflow:
updator["overmidflow"] = histogram["overmidflow"] + (
w if overmidflow else 0.0)
if hasOveroverflow:
updator["overoverflow"] = histogram["overoverflow"] + (
w if overoverflow else 0.0)
if hasNanflow:
updator["nanflow"] = histogram["nanflow"] + (w if nanflow else 0.0)
if hasInfflow:
updator["infflow"] = histogram["infflow"] + (w if infflow else 0.0)
return dict(histogram, **updator)
class Multinomial(LibFcn):
name = prefix + "multinomial"
sig = Sigs([
Sig([{
"datum": P.Array(P.Double())
}, {
"classModel": P.Array(P.Double())
}], P.Double()),
Sig([{
"datum": P.Map(P.Double())
}, {
"classModel": P.Map(P.Double())
}], P.Double()),
Sig([{
"datum": P.Array(P.Double())
}, {
"classModel": P.WildRecord("C", {"values": P.Array(P.Double())})
}], P.Double()),
Sig([{
"datum": P.Map(P.Double())
}, {
"classModel": P.WildRecord("C", {"values": P.Map(P.Double())})
}], P.Double())
])
errcodeBase = 10010
def doarray(self, datum, classModel, pos):
ll = 0.0
normalizing = sum(classModel)
if len(classModel) == 0 or not all(x > 0.0 for x in classModel):
raise PFARuntimeException(
"classModel must be non-empty and strictly positive",
self.errcodeBase + 1, self.name, pos)
if len(datum) != len(classModel):
raise PFARuntimeException("datum and classModel misaligned",
self.errcodeBase + 0, self.name, pos)
for d, p in zip(datum, classModel):
p = p / normalizing + tiny
ll += d * math.log(p)
return ll
def domap(self, datum, classModel, pos):
ll = 0.0
datumkeys = datum.keys()
modelkeys = classModel.keys()
normalizing = sum(classModel.values())
if len(classModel) == 0 or not all(x > 0.0
for x in classModel.values()):
raise PFARuntimeException(
"classModel must be non-empty and strictly positive",
self.errcodeBase + 1, self.name, pos)
if set(datumkeys) != set(modelkeys):
raise PFARuntimeException("datum and classModel misaligned",
self.errcodeBase + 0, self.name, pos)
for d in datumkeys:
p = classModel[d] / normalizing + tiny
ll += datum[d] * math.log(classModel[d])
return ll
def __call__(self, state, scope, pos, paramTypes, datum, classModel):
if paramTypes[1]["type"] == "record":
classModel = classModel["values"]
if paramTypes[0]["type"] == "array":
return self.doarray(datum, classModel, pos)
elif paramTypes[0]["type"] == "map":
return self.domap(datum, classModel, pos)
class FillHistogram(LibFcn):
name = prefix + "fillHistogram"
sig = Sigs([
Sig([{
"x": P.Double()
}, {
"w": P.Double()
}, {
"histogram":
P.WildRecord(
"A", {
"numbins": P.Int(),
"low": P.Double(),
"high": P.Double(),
"values": P.Array(P.Double())
})
}], P.Wildcard("A")),
Sig([{
"x": P.Double()
}, {
"w": P.Double()
}, {
"histogram":
P.WildRecord(
"A", {
"low": P.Double(),
"binsize": P.Double(),
"values": P.Array(P.Double())
})
}], P.Wildcard("A")),
Sig([{
"x": P.Double()
}, {
"w": P.Double()
}, {
"histogram":
P.WildRecord(
"A", {
"ranges": P.Array(P.Array(P.Double())),
"values": P.Array(P.Double())
})
}], P.Wildcard("A"))
])
errcodeBase = 14080
def genpy(self, paramTypes, args, pos):
def has(name, avroType):
for x in paramTypes[2].fields:
if x.name == name:
if not x.avroType.accepts(avroType):
raise PFASemanticException(
"{0} is being given a record type in which the \"{1}\" field is not {2}: {3}"
.format(self.name, name, avroType,
x.avroType), None)
return True
return False
method0 = has("numbins", AvroInt()) and has(
"low", AvroDouble()) and has("high", AvroDouble())
method1 = has("low", AvroDouble()) and has("binsize", AvroDouble())
method2 = has("ranges", AvroArray(AvroArray(AvroDouble())))
if method0 and not method1 and not method2:
method = 0
elif not method0 and method1 and not method2:
method = 1
elif not method0 and not method1 and method2:
method = 2
else:
raise PFASemanticException(
prefix +
"fillHistogram must have \"numbins\", \"low\", \"high\" xor it must have \"low\", \"binsize\" xor it must have \"ranges\", but not any other combination of these fields.",
None)
hasUnderflow = has("underflow", AvroDouble())
hasOverflow = has("overflow", AvroDouble())
hasNanflow = has("nanflow", AvroDouble())
hasInfflow = has("infflow", AvroDouble())
return "self.f[{0}]({1}, {2}, {3}, {4}, {5}, {6})".format(
repr(self.name), ", ".join(
["state", "scope",
repr(pos), repr(paramTypes)] + args), method, hasUnderflow,
hasOverflow, hasNanflow, hasInfflow)
def updateHistogram(self, w, histogram, newValues, hasUnderflow,
hasOverflow, hasNanflow, hasInfflow, underflow,
overflow, nanflow, infflow):
updator = {"values": newValues}
if (hasUnderflow):
updator["underflow"] = histogram["underflow"] + (w if underflow
else 0.0)
if (hasOverflow):
updator["overflow"] = histogram["overflow"] + (w if overflow else
0.0)
if (hasNanflow):
updator["nanflow"] = histogram["nanflow"] + (w if nanflow else 0.0)
if (hasInfflow):
updator["infflow"] = histogram["infflow"] + (w if infflow else 0.0)
return dict(histogram, **updator)
def __call__(self, state, scope, pos, paramTypes, x, w, histogram, method,
hasUnderflow, hasOverflow, hasNanflow, hasInfflow):
values = histogram["values"]
if method == 0:
numbins, low, high = histogram["numbins"], histogram[
"low"], histogram["high"]
if numbins < 1:
raise PFARuntimeException("bad histogram scale",
self.errcodeBase + 2, self.name, pos)
if len(values) != numbins:
raise PFARuntimeException("wrong histogram size",
self.errcodeBase + 0, self.name, pos)
if low >= high or math.isnan(low) or math.isinf(low) or math.isnan(
high) or math.isinf(high):
raise PFARuntimeException("bad histogram range",
self.errcodeBase + 1, self.name, pos)
if hasInfflow and math.isinf(x):
underflow, overflow, nanflow, infflow = False, False, False, True
elif math.isnan(x):
underflow, overflow, nanflow, infflow = False, False, True, False
elif x >= high:
underflow, overflow, nanflow, infflow = False, True, False, False
elif x < low:
underflow, overflow, nanflow, infflow = True, False, False, False
else:
underflow, overflow, nanflow, infflow = False, False, False, False
if not underflow and not overflow and not nanflow and not infflow:
try:
indexFloat = math.floor((x - low) / (high - low) * numbins)
except ZeroDivisionError:
index = 0
else:
if math.isinf(indexFloat):
if indexFloat > 0:
index = INT_MAX_VALUE
else:
index = INT_MIN_VALUE
else:
index = int(indexFloat)
newValues = list(values)
newValues[index] = newValues[index] + w
else:
newValues = values
return self.updateHistogram(w, histogram, newValues, hasUnderflow,
hasOverflow, hasNanflow, hasInfflow,
underflow, overflow, nanflow, infflow)
elif method == 1:
low, binsize = histogram["low"], histogram["binsize"]
if binsize <= 0.0 or math.isnan(binsize) or math.isinf(binsize):
raise PFARuntimeException("bad histogram scale",
self.errcodeBase + 2, self.name, pos)
if hasInfflow and math.isinf(x):
underflow, overflow, nanflow, infflow = False, False, False, True
elif math.isnan(x):
underflow, overflow, nanflow, infflow = False, False, True, False
elif math.isinf(x) and x > 0.0:
underflow, overflow, nanflow, infflow = False, True, False, False
elif x < low:
underflow, overflow, nanflow, infflow = True, False, False, False
else:
underflow, overflow, nanflow, infflow = False, False, False, False
if not underflow and not overflow and not nanflow and not infflow:
currentHigh = low + binsize * len(values)
try:
index = int(
math.floor(
(x - low) / (currentHigh - low) * len(values)))
except ZeroDivisionError:
index = 0
if index < len(values):
newValues = list(values)
newValues[index] = newValues[index] + w
else:
newValues = values + [0.0] * (index - len(values)) + [w]
else:
newValues = values
return self.updateHistogram(w, histogram, newValues, hasUnderflow,
hasOverflow, hasNanflow, hasInfflow,
underflow, overflow, nanflow, infflow)
elif method == 2:
ranges = histogram["ranges"]
if len(values) != len(ranges):
raise PFARuntimeException("wrong histogram size",
self.errcodeBase + 0, self.name, pos)
if any(
len(x) != 2 or x[0] >= x[1] or math.isnan(x[0])
or math.isinf(x[0]) or math.isnan(x[1]) or math.isinf(x[1])
for x in ranges):
raise PFARuntimeException("bad histogram ranges",
self.errcodeBase + 3, self.name, pos)
isInfinite = math.isinf(x)
isNan = math.isnan(x)
newValues = list(values)
hitOne = False
if not isInfinite and not isNan:
for index, rang in enumerate(ranges):
low = rang[0]
high = rang[1]
if low == high and x == low:
newValues[index] = newValues[index] + w
hitOne = True
elif x >= low and x < high:
newValues[index] = newValues[index] + w
hitOne = True
if hasInfflow and isInfinite:
underflow, overflow, nanflow, infflow = False, False, False, True
elif isNan:
underflow, overflow, nanflow, infflow = False, False, True, False
elif not hitOne:
underflow, overflow, nanflow, infflow = True, False, False, False
else:
underflow, overflow, nanflow, infflow = False, False, False, False
return self.updateHistogram(w, histogram, newValues, hasUnderflow,
hasOverflow, hasNanflow, hasInfflow,
underflow, overflow, nanflow, infflow)
class ForecastHoltWinters(LibFcn):
name = prefix + "forecastHoltWinters"
sig = Sig(
[{
"n": P.Int()
}, {
"state": P.WildRecord("A", {
"level": P.Double(),
"trend": P.Double()
})
}], P.Array(P.Double()))
errcodeBase = 14070
def genpy(self, paramTypes, args, pos):
hasCycle = False
hasMultiplicative = False
for x in paramTypes[1].fields:
if x.name == "cycle":
if not x.avroType.accepts(AvroArray(AvroDouble())):
raise PFASemanticException(
self.name +
" is being given a state record type in which the \"cycle\" field is not an array of double: "
+ str(x.avroType), None)
hasCycle = True
elif x.name == "multiplicative":
if not x.avroType.accepts(AvroBoolean()):
raise PFASemanticException(
self.name +
" is being given a state record type in which the \"multiplicative\" field is not a boolean: "
+ str(x.avroType), None)
hasMultiplicative = True
if hasCycle ^ hasMultiplicative:
raise PFASemanticException(
self.name +
" is being given a state record type with a \"cycle\" but no \"multiplicative\" or vice-versa",
None)
return "self.f[{0}]({1}, {2})".format(
repr(self.name), ", ".join(
["state", "scope",
repr(pos), repr(paramTypes)] + args), hasCycle)
def __call__(self, state, scope, pos, paramTypes, n, theState,
hasPeriodic):
level = theState["level"]
trend = theState["trend"]
if not hasPeriodic:
return [level + i * trend for i in range(1, n + 1)]
else:
cycle = theState["cycle"]
L = len(cycle)
if L == 0:
raise PFARuntimeException("empty cycle", self.errcodeBase + 0,
self.name, pos)
if theState["multiplicative"]:
return [(level + i * trend) * cycle[i % L]
for i in range(1, n + 1)]
else:
return [
level + i * trend + cycle[i % L] for i in range(1, n + 1)
]
class UpdateHoltWintersPeriodic(LibFcn):
name = prefix + "updateHoltWintersPeriodic"
sig = Sig([{
"x": P.Double()
}, {
"alpha": P.Double()
}, {
"beta": P.Double()
}, {
"gamma": P.Double()
}, {
"state":
P.WildRecord(
"A", {
"level": P.Double(),
"trend": P.Double(),
"cycle": P.Array(P.Double()),
"multiplicative": P.Boolean()
})
}], P.Wildcard("A"))
errcodeBase = 14050
def __call__(self, state, scope, pos, paramTypes, x, alpha, beta, gamma,
theState):
if alpha < 0.0 or alpha > 1.0:
raise PFARuntimeException("alpha out of range",
self.errcodeBase + 0, self.name, pos)
if beta < 0.0 or beta > 1.0:
raise PFARuntimeException("beta out of range",
self.errcodeBase + 1, self.name, pos)
if gamma < 0.0 or gamma > 1.0:
raise PFARuntimeException("gamma out of range",
self.errcodeBase + 2, self.name, pos)
level_prev = theState["level"]
trend_prev = theState["trend"]
cycle_unrotated = theState["cycle"]
if len(cycle_unrotated) == 0:
raise PFARuntimeException("empty cycle", self.errcodeBase + 3,
self.name, pos)
cycle_rotated = cycle_unrotated[1:] + [cycle_unrotated[0]]
cycle_prev = cycle_rotated[0]
if theState["multiplicative"]:
level = div(alpha * x,
cycle_prev) + (1.0 - alpha) * (level_prev + trend_prev)
trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev
cycle = div(gamma * x, level) + (1.0 - gamma) * cycle_prev
else:
level = alpha * (x - cycle_prev) + (1.0 - alpha) * (level_prev +
trend_prev)
trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev
cycle = gamma * (x - level) + (1.0 - gamma) * cycle_prev
return dict(theState,
level=level,
trend=trend,
cycle=([cycle] + cycle_rotated[1:]))
class UpdateWindow(UpdateMeanVariance):
name = prefix + "updateWindow"
sig = Sig([{
"x": P.Double()
}, {
"w": P.Double()
}, {
"state":
P.Array(
P.WildRecord("A", {
"x": P.Double(),
"w": P.Double(),
"count": P.Double()
}))
}, {
"windowSize": P.Int()
}], P.Array(P.Wildcard("A")))
errcodeBase = 14020
def _getRecord(self, paramType):
return paramType.items
def __call__(self, state, scope, pos, paramTypes, x, w, theState,
windowSize, level):
if windowSize < 2:
raise PFARuntimeException("windowSize must be at least 2",
self.errcodeBase + 0, self.name, pos)
if len(theState) == 0:
out = {}
paramType = state.parser.getAvroType(paramTypes[2])
for field in self._getRecord(paramType).fields:
if field.name == "x":
out["x"] = x
elif field.name == "w":
out["w"] = w
elif field.name == "count":
out["count"] = w
elif field.name == "mean":
out["mean"] = x
elif field.name == "variance":
out["variance"] = 0.0
else:
raise PFARuntimeException(
"cannot initialize unrecognized fields",
self.errcodeBase + 1, self.name, pos)
return [out]
elif len(theState) >= windowSize:
record = theState[-1]
splitAt = len(theState) - windowSize + 1
remove, keep = theState[:splitAt], theState[splitAt:]
oldx = [xi["x"] for xi in remove]
oldw = [-xi["w"] for xi in remove]
originalCount = record["count"]
count = originalCount + w
count2 = count + sum(oldw)
if level == 0:
return keep + [dict(record, x=x, w=w, count=count2)]
else:
mean = record["mean"]
delta = x - mean
shift = div(delta * w, count)
mean += shift
accumulatedCount = count
varianceCorrection = 0.0
for ox, ow in zip(oldx, oldw):
accumulatedCount += ow
delta2 = ox - mean
shift2 = div(delta2 * ow, accumulatedCount)
mean += shift2
varianceCorrection += (accumulatedCount -
ow) * delta2 * shift2
if level == 1:
return keep + [
dict(record, x=x, w=w, count=count2, mean=mean)
]
else:
varianceTimesCount = record["variance"] * originalCount
varianceTimesCount += originalCount * delta * shift
varianceTimesCount += varianceCorrection
return keep + [
dict(record,
x=x,
w=w,
count=count2,
mean=mean,
variance=div(varianceTimesCount, count2))
]
else:
record = theState[-1]
originalCount = record["count"]
count = originalCount + w
if level == 0:
return theState + [dict(record, x=x, w=w, count=count)]
else:
mean = record["mean"]
delta = x - mean
shift = div(delta * w, count)
mean += shift
if level == 1:
return theState + [
dict(record, x=x, w=w, count=count, mean=mean)
]
else:
varianceTimesCount = record["variance"] * originalCount
varianceTimesCount += originalCount * delta * shift
return theState + [
dict(record,
x=x,
w=w,
count=count,
mean=mean,
variance=div(varianceTimesCount, count))
]
class UpdateCovariance(LibFcn):
name = prefix + "updateCovariance"
sig = Sigs([
Sig([{
"x": P.Array(P.Double())
}, {
"w": P.Double()
}, {
"state":
P.WildRecord(
"A", {
"count": P.Double(),
"mean": P.Array(P.Double()),
"covariance": P.Array(P.Array(P.Double()))
})
}], P.Wildcard("A")),
Sig([{
"x": P.Map(P.Double())
}, {
"w": P.Double()
}, {
"state":
P.WildRecord(
"A", {
"count": P.Map(P.Map(P.Double())),
"mean": P.Map(P.Double()),
"covariance": P.Map(P.Map(P.Double()))
})
}], P.Wildcard("A"))
])
errcodeBase = 14010
def __call__(self, state, scope, pos, paramTypes, x, w, theState):
size = len(x)
if size < 2:
raise PFARuntimeException("too few components",
self.errcodeBase + 1, self.name, pos)
if paramTypes[0]["type"] == "map":
oldCount = theState["count"]
oldMean = theState["mean"]
oldCovariance = theState["covariance"]
countKeys = set(oldCount.keys()).union(
reduce(lambda a, b: a.union(b),
(set(v.keys()) for v in oldCount.values()), set()))
covarKeys = set(oldCovariance.keys()).union(
reduce(lambda a, b: a.union(b),
(set(v.keys()) for v in oldCovariance.values()), set()))
keys = set(x.keys()).union(countKeys).union(covarKeys)
newCount = {}
for i in keys:
row = {}
for j in keys:
old = oldCount.get(i, {}).get(j, 0.0)
if (i in x) and (j in x):
row[j] = old + w
else:
row[j] = old
newCount[i] = row
newMean = {}
for i in keys:
old = oldMean.get(i, 0.0)
if i in x:
newMean[i] = old + div((x[i] - old) * w, newCount[i][i])
else:
newMean[i] = old
newCovariance = {}
for i in keys:
row = {}
for j in keys:
oldCov = oldCovariance.get(i, {}).get(j, 0.0)
if (i in x) and (j in x):
oldC = oldCount.get(i, {}).get(j, 0.0)
oldMi = oldMean.get(i, 0.0)
oldMj = oldMean.get(j, 0.0)
row[j] = div(((oldCov * oldC) + div(
(x[i] - oldMi) *
(x[j] - oldMj) * w * oldC, newCount[i][j])),
newCount[i][j])
else:
row[j] = oldCov
newCovariance[i] = row
return dict(theState,
count=newCount,
mean=newMean,
covariance=newCovariance)
else:
oldCount = theState["count"]
oldMean = theState["mean"]
oldCovariance = theState["covariance"]
if (size != len(oldMean) or size != len(oldCovariance)
or any(size != len(xi) for xi in oldCovariance)):
raise PFARuntimeException("unequal length arrays",
self.errcodeBase + 2, self.name, pos)
newCount = oldCount + w
newMean = [
oldm + div((x[i] - oldm) * w, newCount)
for i, oldm in enumerate(oldMean)
]
newCovariance = []
for i in range(size):
row = []
for j in range(size):
row.append(
div((oldCovariance[i][j] * oldCount) + div(
(x[i] - oldMean[i]) *
(x[j] - oldMean[j]) * w * oldCount, newCount),
newCount))
newCovariance.append(row)
return dict(theState,
count=newCount,
mean=newMean,
covariance=newCovariance)
