def assignRet(self, pat, assignments): """Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string" etc.) to the return pattern. :type pat: titus.P :param pat: original return pattern :type assignments: dict from label letters to titus.datatype.AvroType :param assignments: assigned types to apply :rtype: titus.datatype.AvroType :return: resolved type for the return value of the signature """ if isinstance(pat, P.Null): return AvroNull() elif isinstance(pat, P.Boolean): return AvroBoolean() elif isinstance(pat, P.Int): return AvroInt() elif isinstance(pat, P.Long): return AvroLong() elif isinstance(pat, P.Float): return AvroFloat() elif isinstance(pat, P.Double): return AvroDouble() elif isinstance(pat, P.Bytes): return AvroBytes() elif isinstance(pat, P.String): return AvroString() elif isinstance(pat, P.Array): return AvroArray(self.assignRet(pat.items, assignments)) elif isinstance(pat, P.Map): return AvroMap(self.assignRet(pat.values, assignments)) elif isinstance(pat, P.Union): return AvroUnion([self.assignRet(x, assignments) for x in pat.types]) elif isinstance(pat, P.Fixed): return P.toType(pat) elif isinstance(pat, P.Enum): return P.toType(pat) elif isinstance(pat, P.Record): return P.toType(pat) elif isinstance(pat, P.Fcn): return P.toType(pat) elif isinstance(pat, P.Wildcard): return assignments[pat.label] elif isinstance(pat, P.WildRecord): return assignments[pat.label] elif isinstance(pat, P.WildEnum): return assignments[pat.label] elif isinstance(pat, P.WildFixed): return assignments[pat.label] elif isinstance(pat, P.EnumFields): return assignments[pat.label] else: raise Exception(repr(pat))
def assign(self, pat, arg, assignments): if isinstance(pat, P.Null) and isinstance(arg, AvroNull): return arg elif isinstance(pat, P.Boolean) and isinstance(arg, AvroBoolean): return arg elif isinstance(pat, P.Int) and isinstance(arg, AvroInt): return AvroInt() elif isinstance(pat, P.Long) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong)): return AvroLong() elif isinstance(pat, P.Float) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat)): return AvroFloat() elif isinstance(pat, P.Double) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat) or isinstance(arg, AvroDouble)): return AvroDouble() elif isinstance(pat, P.Bytes) and isinstance(arg, AvroBytes): return arg elif isinstance(pat, P.String) and isinstance(arg, AvroString): return arg elif isinstance(pat, P.Array) and isinstance(arg, AvroArray): return AvroArray(P.mustBeAvro(self.assign(pat.items, arg.items, assignments))) elif isinstance(pat, P.Map) and isinstance(arg, AvroMap): return AvroMap(P.mustBeAvro(self.assign(pat.values, arg.values, assignments))) elif isinstance(pat, P.Union) and isinstance(arg, AvroUnion): return arg elif isinstance(pat, P.Union) and isinstance(arg, AvroType): return arg elif isinstance(pat, P.Fixed) and isinstance(arg, AvroFixed): return arg elif isinstance(pat, P.Enum) and isinstance(arg, AvroEnum): return arg elif isinstance(pat, P.Record) and isinstance(arg, AvroRecord): return arg elif isinstance(pat, P.Fcn) and isinstance(arg, FcnType): return arg elif isinstance(pat, P.Wildcard): return assignments[pat.label] elif isinstance(pat, P.WildRecord): return assignments[pat.label] elif isinstance(pat, P.WildEnum): return assignments[pat.label] elif isinstance(pat, P.WildFixed): return assignments[pat.label] elif isinstance(pat, P.EnumFields): return assignments[pat.label] else: raise Exception
def assignRet(self, pat, assignments): if isinstance(pat, P.Null): return AvroNull() elif isinstance(pat, P.Boolean): return AvroBoolean() elif isinstance(pat, P.Int): return AvroInt() elif isinstance(pat, P.Long): return AvroLong() elif isinstance(pat, P.Float): return AvroFloat() elif isinstance(pat, P.Double): return AvroDouble() elif isinstance(pat, P.Bytes): return AvroBytes() elif isinstance(pat, P.String): return AvroString() elif isinstance(pat, P.Array): return AvroArray(self.assignRet(pat.items, assignments)) elif isinstance(pat, P.Map): return AvroMap(self.assignRet(pat.values, assignments)) elif isinstance(pat, P.Union): return AvroUnion([self.assignRet(x, assignments) for x in pat.types]) elif isinstance(pat, P.Fixed): return P.toType(pat) elif isinstance(pat, P.Enum): return P.toType(pat) elif isinstance(pat, P.Record): return P.toType(pat) elif isinstance(pat, P.Fcn): return P.toType(pat) elif isinstance(pat, P.Wildcard): return assignments[pat.label] elif isinstance(pat, P.WildRecord): return assignments[pat.label] elif isinstance(pat, P.WildEnum): return assignments[pat.label] elif isinstance(pat, P.WildFixed): return assignments[pat.label] elif isinstance(pat, P.EnumFields): return assignments[pat.label] else: raise Exception(repr(pat))
def assign(self, pat, arg, assignments): """Apply the label assignments (e.g. "A" matched to "int", "B" matched to "string", etc.) to each parameter of the signature. :type pat: titus.P :param pat: original parameter pattern :type arg: titus.datatype.AvroType :param arg: supplied argument type :type assignments: dict from label letters to titus.datatype.AvroType :param assignments: assigned types to apply :rtype: titus.datatype.AvroType :return: resolved type for one parameter of the signature """ if isinstance(pat, P.Null) and isinstance(arg, AvroNull): return arg elif isinstance(pat, P.Boolean) and isinstance(arg, AvroBoolean): return arg elif isinstance(pat, P.Int) and isinstance(arg, AvroInt): return AvroInt() elif isinstance(pat, P.Long) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong)): return AvroLong() elif isinstance(pat, P.Float) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat)): return AvroFloat() elif isinstance(pat, P.Double) and (isinstance(arg, AvroInt) or isinstance(arg, AvroLong) or isinstance(arg, AvroFloat) or isinstance(arg, AvroDouble)): return AvroDouble() elif isinstance(pat, P.Bytes) and isinstance(arg, AvroBytes): return arg elif isinstance(pat, P.String) and isinstance(arg, AvroString): return arg elif isinstance(pat, P.Array) and isinstance(arg, AvroArray): return AvroArray(P.mustBeAvro(self.assign(pat.items, arg.items, assignments))) elif isinstance(pat, P.Map) and isinstance(arg, AvroMap): return AvroMap(P.mustBeAvro(self.assign(pat.values, arg.values, assignments))) elif isinstance(pat, P.Union) and isinstance(arg, AvroUnion): return arg elif isinstance(pat, P.Union) and isinstance(arg, AvroType): return arg elif isinstance(pat, P.Fixed) and isinstance(arg, AvroFixed): return arg elif isinstance(pat, P.Enum) and isinstance(arg, AvroEnum): return arg elif isinstance(pat, P.Record) and isinstance(arg, AvroRecord): return arg elif isinstance(pat, P.Fcn) and isinstance(arg, FcnType): return arg elif isinstance(pat, P.Wildcard): return assignments[pat.label] elif isinstance(pat, P.WildRecord): return assignments[pat.label] elif isinstance(pat, P.WildEnum): return assignments[pat.label] elif isinstance(pat, P.WildFixed): return assignments[pat.label] elif isinstance(pat, P.EnumFields): return assignments[pat.label] else: raise Exception
def broadestType(candidates): """Compute the narrowest possible supertype of a set of types. :type candidates: list of titus.datatype.AvroType :param candidates: set of types for which to find the narrowest possible supertype :rtype: titus.datatype.AvroType :return: narrowest possible supertype, usually a union of the candidates """ realCandidates = [x for x in candidates if not isinstance(x, ExceptionType)] if len(candidates) == 0: return ValueError("empty list of types") elif len(realCandidates) == 0: return ValueError("list of types consists only of exception type") elif all(isinstance(x, AvroNull) for x in realCandidates): return realCandidates[0] elif all(isinstance(x, AvroBoolean) for x in realCandidates): return realCandidates[0] elif all(isinstance(x, AvroInt) for x in realCandidates): return AvroInt() elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) for x in realCandidates): return AvroLong() elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) or isinstance(x, AvroFloat) for x in realCandidates): return AvroFloat() elif all(isinstance(x, AvroInt) or isinstance(x, AvroLong) or isinstance(x, AvroFloat) or isinstance(x, AvroDouble) for x in realCandidates): return AvroDouble() elif all(isinstance(x, AvroBytes) for x in realCandidates): return realCandidates[0] elif all(isinstance(x, AvroString) for x in realCandidates): return realCandidates[0] elif all(isinstance(x, AvroArray) for x in realCandidates): return AvroArray(P.mustBeAvro(LabelData.broadestType([x.items for x in realCandidates]))) elif all(isinstance(x, AvroMap) for x in realCandidates): return AvroMap(P.mustBeAvro(LabelData.broadestType([x.values for x in realCandidates]))) elif all(isinstance(x, AvroFixed) for x in realCandidates): fullName = realCandidates[0].fullName if all(x.fullName == fullName for x in realCandidates[1:]): return realCandidates[0] else: raise IncompatibleTypes("incompatible fixed types: " + " ".join(map(repr, realCandidates))) elif all(isinstance(x, AvroEnum) for x in realCandidates): fullName = realCandidates[0].fullName if all(x.fullName == fullName for x in realCandidates[1:]): return realCandidates[0] else: raise IncompatibleTypes("incompatible enum types: " + " ".join(map(repr, realCandidates))) elif all(isinstance(x, AvroRecord) for x in realCandidates): fullName = realCandidates[0].fullName if all(x.fullName == fullName for x in realCandidates[1:]): return realCandidates[0] else: raise IncompatibleTypes("incompatible record types: " + " ".join(map(repr, realCandidates))) elif all(isinstance(x, FcnType) for x in realCandidates): params = realCandidates[0].params ret = realCandidates[0].ret if all(x.params == params and x.ret == ret for x in realCandidates[1:]): return realCandidates[0] else: raise IncompatibleTypes("incompatible function types: " + " ".join(map(repr, realCandidates))) elif not any(isinstance(x, FcnType) for x in realCandidates): types = LabelData.distinctTypes(realCandidates) types = [P.mustBeAvro(x) for x in types] countFixed = 0 countEnum = 0 for t in types: if isinstance(t, AvroFixed): countFixed += 1 if isinstance(t, AvroEnum): countEnum += 1 if countFixed > 1: raise IncompatibleTypes("incompatible fixed types") if countEnum > 1: raise IncompatibleTypes("incompatible enum types") return AvroUnion(types) else: raise IncompatibleTypes("incompatible function/non-function types: " + " ".join(map(repr, realCandidates)))
class Bernoulli(LibFcn): name = prefix + "bernoulli" sig = Sigs([ Sig([{ "datum": P.Array(P.String()) }, { "classModel": P.Map(P.Double()) }], P.Double()), Sig([{ "datum": P.Array(P.String()) }, { "classModel": P.WildRecord("C", {"values": P.Map(P.Double())}) }], P.Double()) ]) errcodeBase = 10020 def __call__(self, state, scope, pos, paramTypes, datum, classModel): if paramTypes[1]["type"] == "record": classModel = classModel["values"] ll = 0.0 for v in classModel.values(): if (v <= 0.0) or (v >= 1.0): raise PFARuntimeException( "probability in classModel cannot be less than 0 or greater than 1", self.errcodeBase + 0, self.name, pos) ll += math.log(1.0 - v) for item in datum: p = classModel.get(item, None) if p is not None: ll += math.log(p) - math.log(1.0 - p) return ll
class FillHistogram2d(LibFcn): name = prefix + "fillHistogram2d" sig = Sig([{ "x": P.Double() }, { "y": P.Double() }, { "w": P.Double() }, { "histogram": P.WildRecord( "A", { "xnumbins": P.Int(), "xlow": P.Double(), "xhigh": P.Double(), "ynumbins": P.Int(), "ylow": P.Double(), "yhigh": P.Double(), "values": P.Array(P.Array(P.Double())) }) }], P.Wildcard("A")) errcodeBase = 14090 def genpy(self, paramTypes, args, pos): def has(name): for x in paramTypes[3].fields: if x.name == name: if not x.avroType.accepts(AvroDouble()): raise PFASemanticException( "{0} is being given a record type in which the \"{1}\" field is not a double: {2}" .format(self.name, name, x.avroType), None) return True return False return "self.f[{0}]({1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11})".format( repr(self.name), ", ".join( ["state", "scope", repr(pos), repr(paramTypes)] + args), has("underunderflow"), has("undermidflow"), has("underoverflow"), has("midunderflow"), has("midoverflow"), has("overunderflow"), has("overmidflow"), has("overoverflow"), has("nanflow"), has("infflow")) def __call__(self, state, scope, pos, paramTypes, x, y, w, histogram, hasUnderunderflow, hasUndermidflow, hasUnderoverflow, hasMidunderflow, hasMidoverflow, hasOverunderflow, hasOvermidflow, hasOveroverflow, hasNanflow, hasInfflow): values = histogram["values"] xnumbins = histogram["xnumbins"] xlow = histogram["xlow"] xhigh = histogram["xhigh"] ynumbins = histogram["ynumbins"] ylow = histogram["ylow"] yhigh = histogram["yhigh"] if xnumbins < 1 or ynumbins < 1: raise PFARuntimeException("bad histogram scale", self.errcodeBase + 2, self.name, pos) if len(values) != xnumbins or any(len(x) != ynumbins for x in values): raise PFARuntimeException("wrong histogram size", self.errcodeBase + 0, self.name, pos) if xlow >= xhigh or ylow >= yhigh or math.isnan(xlow) or math.isinf( xlow) or math.isnan(xhigh) or math.isinf(xhigh) or math.isnan( ylow) or math.isinf(ylow) or math.isnan( yhigh) or math.isinf(yhigh): raise PFARuntimeException("bad histogram range", self.errcodeBase + 1, self.name, pos) if math.isnan(x) or math.isnan( y): # do nan check first: nan wins over inf underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, True, False elif hasInfflow and (math.isinf(x) or math.isinf(y)): underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, False, True elif x >= xhigh and y >= yhigh: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, True, False, False elif x >= xhigh and y >= ylow and y < yhigh: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, True, False, False, False elif x >= xhigh and y < ylow: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, True, False, False, False, False elif x >= xlow and x < xhigh and y >= yhigh: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, True, False, False, False, False, False elif x >= xlow and x < xhigh and y < ylow: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, True, False, False, False, False, False, False elif x < xlow and y >= yhigh: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, True, False, False, False, False, False, False, False elif x < xlow and y >= ylow and y < yhigh: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, True, False, False, False, False, False, False, False, False elif x < xlow and y < ylow: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = True, False, False, False, False, False, False, False, False, False else: underunderflow, undermidflow, underoverflow, midunderflow, midoverflow, overunderflow, overmidflow, overoverflow, nanflow, infflow = False, False, False, False, False, False, False, False, False, False if not underunderflow and not undermidflow and not underoverflow and not midunderflow and not midoverflow and not overunderflow and not overmidflow and not overoverflow and not nanflow and not infflow: try: xindex = int(math.floor( (x - xlow) / (xhigh - xlow) * xnumbins)) except ZeroDivisionError: xindex = 0 try: yindex = int(math.floor( (y - ylow) / (yhigh - ylow) * ynumbins)) except ZeroDivisionError: yindex = 0 newValues = [list(x) for x in values] newValues[xindex][yindex] = newValues[xindex][yindex] + w else: newValues = values updator = {"values": newValues} if hasUnderunderflow: updator["underunderflow"] = histogram["underunderflow"] + ( w if underunderflow else 0.0) if hasUndermidflow: updator["undermidflow"] = histogram["undermidflow"] + ( w if undermidflow else 0.0) if hasUnderoverflow: updator["underoverflow"] = histogram["underoverflow"] + ( w if underoverflow else 0.0) if hasMidunderflow: updator["midunderflow"] = histogram["midunderflow"] + ( w if midunderflow else 0.0) if hasMidoverflow: updator["midoverflow"] = histogram["midoverflow"] + ( w if midoverflow else 0.0) if hasOverunderflow: updator["overunderflow"] = histogram["overunderflow"] + ( w if overunderflow else 0.0) if hasOvermidflow: updator["overmidflow"] = histogram["overmidflow"] + ( w if overmidflow else 0.0) if hasOveroverflow: updator["overoverflow"] = histogram["overoverflow"] + ( w if overoverflow else 0.0) if hasNanflow: updator["nanflow"] = histogram["nanflow"] + (w if nanflow else 0.0) if hasInfflow: updator["infflow"] = histogram["infflow"] + (w if infflow else 0.0) return dict(histogram, **updator)
class Multinomial(LibFcn): name = prefix + "multinomial" sig = Sigs([ Sig([{ "datum": P.Array(P.Double()) }, { "classModel": P.Array(P.Double()) }], P.Double()), Sig([{ "datum": P.Map(P.Double()) }, { "classModel": P.Map(P.Double()) }], P.Double()), Sig([{ "datum": P.Array(P.Double()) }, { "classModel": P.WildRecord("C", {"values": P.Array(P.Double())}) }], P.Double()), Sig([{ "datum": P.Map(P.Double()) }, { "classModel": P.WildRecord("C", {"values": P.Map(P.Double())}) }], P.Double()) ]) errcodeBase = 10010 def doarray(self, datum, classModel, pos): ll = 0.0 normalizing = sum(classModel) if len(classModel) == 0 or not all(x > 0.0 for x in classModel): raise PFARuntimeException( "classModel must be non-empty and strictly positive", self.errcodeBase + 1, self.name, pos) if len(datum) != len(classModel): raise PFARuntimeException("datum and classModel misaligned", self.errcodeBase + 0, self.name, pos) for d, p in zip(datum, classModel): p = p / normalizing + tiny ll += d * math.log(p) return ll def domap(self, datum, classModel, pos): ll = 0.0 datumkeys = datum.keys() modelkeys = classModel.keys() normalizing = sum(classModel.values()) if len(classModel) == 0 or not all(x > 0.0 for x in classModel.values()): raise PFARuntimeException( "classModel must be non-empty and strictly positive", self.errcodeBase + 1, self.name, pos) if set(datumkeys) != set(modelkeys): raise PFARuntimeException("datum and classModel misaligned", self.errcodeBase + 0, self.name, pos) for d in datumkeys: p = classModel[d] / normalizing + tiny ll += datum[d] * math.log(classModel[d]) return ll def __call__(self, state, scope, pos, paramTypes, datum, classModel): if paramTypes[1]["type"] == "record": classModel = classModel["values"] if paramTypes[0]["type"] == "array": return self.doarray(datum, classModel, pos) elif paramTypes[0]["type"] == "map": return self.domap(datum, classModel, pos)
class FillHistogram(LibFcn): name = prefix + "fillHistogram" sig = Sigs([ Sig([{ "x": P.Double() }, { "w": P.Double() }, { "histogram": P.WildRecord( "A", { "numbins": P.Int(), "low": P.Double(), "high": P.Double(), "values": P.Array(P.Double()) }) }], P.Wildcard("A")), Sig([{ "x": P.Double() }, { "w": P.Double() }, { "histogram": P.WildRecord( "A", { "low": P.Double(), "binsize": P.Double(), "values": P.Array(P.Double()) }) }], P.Wildcard("A")), Sig([{ "x": P.Double() }, { "w": P.Double() }, { "histogram": P.WildRecord( "A", { "ranges": P.Array(P.Array(P.Double())), "values": P.Array(P.Double()) }) }], P.Wildcard("A")) ]) errcodeBase = 14080 def genpy(self, paramTypes, args, pos): def has(name, avroType): for x in paramTypes[2].fields: if x.name == name: if not x.avroType.accepts(avroType): raise PFASemanticException( "{0} is being given a record type in which the \"{1}\" field is not {2}: {3}" .format(self.name, name, avroType, x.avroType), None) return True return False method0 = has("numbins", AvroInt()) and has( "low", AvroDouble()) and has("high", AvroDouble()) method1 = has("low", AvroDouble()) and has("binsize", AvroDouble()) method2 = has("ranges", AvroArray(AvroArray(AvroDouble()))) if method0 and not method1 and not method2: method = 0 elif not method0 and method1 and not method2: method = 1 elif not method0 and not method1 and method2: method = 2 else: raise PFASemanticException( prefix + "fillHistogram must have \"numbins\", \"low\", \"high\" xor it must have \"low\", \"binsize\" xor it must have \"ranges\", but not any other combination of these fields.", None) hasUnderflow = has("underflow", AvroDouble()) hasOverflow = has("overflow", AvroDouble()) hasNanflow = has("nanflow", AvroDouble()) hasInfflow = has("infflow", AvroDouble()) return "self.f[{0}]({1}, {2}, {3}, {4}, {5}, {6})".format( repr(self.name), ", ".join( ["state", "scope", repr(pos), repr(paramTypes)] + args), method, hasUnderflow, hasOverflow, hasNanflow, hasInfflow) def updateHistogram(self, w, histogram, newValues, hasUnderflow, hasOverflow, hasNanflow, hasInfflow, underflow, overflow, nanflow, infflow): updator = {"values": newValues} if (hasUnderflow): updator["underflow"] = histogram["underflow"] + (w if underflow else 0.0) if (hasOverflow): updator["overflow"] = histogram["overflow"] + (w if overflow else 0.0) if (hasNanflow): updator["nanflow"] = histogram["nanflow"] + (w if nanflow else 0.0) if (hasInfflow): updator["infflow"] = histogram["infflow"] + (w if infflow else 0.0) return dict(histogram, **updator) def __call__(self, state, scope, pos, paramTypes, x, w, histogram, method, hasUnderflow, hasOverflow, hasNanflow, hasInfflow): values = histogram["values"] if method == 0: numbins, low, high = histogram["numbins"], histogram[ "low"], histogram["high"] if numbins < 1: raise PFARuntimeException("bad histogram scale", self.errcodeBase + 2, self.name, pos) if len(values) != numbins: raise PFARuntimeException("wrong histogram size", self.errcodeBase + 0, self.name, pos) if low >= high or math.isnan(low) or math.isinf(low) or math.isnan( high) or math.isinf(high): raise PFARuntimeException("bad histogram range", self.errcodeBase + 1, self.name, pos) if hasInfflow and math.isinf(x): underflow, overflow, nanflow, infflow = False, False, False, True elif math.isnan(x): underflow, overflow, nanflow, infflow = False, False, True, False elif x >= high: underflow, overflow, nanflow, infflow = False, True, False, False elif x < low: underflow, overflow, nanflow, infflow = True, False, False, False else: underflow, overflow, nanflow, infflow = False, False, False, False if not underflow and not overflow and not nanflow and not infflow: try: indexFloat = math.floor((x - low) / (high - low) * numbins) except ZeroDivisionError: index = 0 else: if math.isinf(indexFloat): if indexFloat > 0: index = INT_MAX_VALUE else: index = INT_MIN_VALUE else: index = int(indexFloat) newValues = list(values) newValues[index] = newValues[index] + w else: newValues = values return self.updateHistogram(w, histogram, newValues, hasUnderflow, hasOverflow, hasNanflow, hasInfflow, underflow, overflow, nanflow, infflow) elif method == 1: low, binsize = histogram["low"], histogram["binsize"] if binsize <= 0.0 or math.isnan(binsize) or math.isinf(binsize): raise PFARuntimeException("bad histogram scale", self.errcodeBase + 2, self.name, pos) if hasInfflow and math.isinf(x): underflow, overflow, nanflow, infflow = False, False, False, True elif math.isnan(x): underflow, overflow, nanflow, infflow = False, False, True, False elif math.isinf(x) and x > 0.0: underflow, overflow, nanflow, infflow = False, True, False, False elif x < low: underflow, overflow, nanflow, infflow = True, False, False, False else: underflow, overflow, nanflow, infflow = False, False, False, False if not underflow and not overflow and not nanflow and not infflow: currentHigh = low + binsize * len(values) try: index = int( math.floor( (x - low) / (currentHigh - low) * len(values))) except ZeroDivisionError: index = 0 if index < len(values): newValues = list(values) newValues[index] = newValues[index] + w else: newValues = values + [0.0] * (index - len(values)) + [w] else: newValues = values return self.updateHistogram(w, histogram, newValues, hasUnderflow, hasOverflow, hasNanflow, hasInfflow, underflow, overflow, nanflow, infflow) elif method == 2: ranges = histogram["ranges"] if len(values) != len(ranges): raise PFARuntimeException("wrong histogram size", self.errcodeBase + 0, self.name, pos) if any( len(x) != 2 or x[0] >= x[1] or math.isnan(x[0]) or math.isinf(x[0]) or math.isnan(x[1]) or math.isinf(x[1]) for x in ranges): raise PFARuntimeException("bad histogram ranges", self.errcodeBase + 3, self.name, pos) isInfinite = math.isinf(x) isNan = math.isnan(x) newValues = list(values) hitOne = False if not isInfinite and not isNan: for index, rang in enumerate(ranges): low = rang[0] high = rang[1] if low == high and x == low: newValues[index] = newValues[index] + w hitOne = True elif x >= low and x < high: newValues[index] = newValues[index] + w hitOne = True if hasInfflow and isInfinite: underflow, overflow, nanflow, infflow = False, False, False, True elif isNan: underflow, overflow, nanflow, infflow = False, False, True, False elif not hitOne: underflow, overflow, nanflow, infflow = True, False, False, False else: underflow, overflow, nanflow, infflow = False, False, False, False return self.updateHistogram(w, histogram, newValues, hasUnderflow, hasOverflow, hasNanflow, hasInfflow, underflow, overflow, nanflow, infflow)
class ForecastHoltWinters(LibFcn): name = prefix + "forecastHoltWinters" sig = Sig( [{ "n": P.Int() }, { "state": P.WildRecord("A", { "level": P.Double(), "trend": P.Double() }) }], P.Array(P.Double())) errcodeBase = 14070 def genpy(self, paramTypes, args, pos): hasCycle = False hasMultiplicative = False for x in paramTypes[1].fields: if x.name == "cycle": if not x.avroType.accepts(AvroArray(AvroDouble())): raise PFASemanticException( self.name + " is being given a state record type in which the \"cycle\" field is not an array of double: " + str(x.avroType), None) hasCycle = True elif x.name == "multiplicative": if not x.avroType.accepts(AvroBoolean()): raise PFASemanticException( self.name + " is being given a state record type in which the \"multiplicative\" field is not a boolean: " + str(x.avroType), None) hasMultiplicative = True if hasCycle ^ hasMultiplicative: raise PFASemanticException( self.name + " is being given a state record type with a \"cycle\" but no \"multiplicative\" or vice-versa", None) return "self.f[{0}]({1}, {2})".format( repr(self.name), ", ".join( ["state", "scope", repr(pos), repr(paramTypes)] + args), hasCycle) def __call__(self, state, scope, pos, paramTypes, n, theState, hasPeriodic): level = theState["level"] trend = theState["trend"] if not hasPeriodic: return [level + i * trend for i in range(1, n + 1)] else: cycle = theState["cycle"] L = len(cycle) if L == 0: raise PFARuntimeException("empty cycle", self.errcodeBase + 0, self.name, pos) if theState["multiplicative"]: return [(level + i * trend) * cycle[i % L] for i in range(1, n + 1)] else: return [ level + i * trend + cycle[i % L] for i in range(1, n + 1) ]
class UpdateHoltWintersPeriodic(LibFcn): name = prefix + "updateHoltWintersPeriodic" sig = Sig([{ "x": P.Double() }, { "alpha": P.Double() }, { "beta": P.Double() }, { "gamma": P.Double() }, { "state": P.WildRecord( "A", { "level": P.Double(), "trend": P.Double(), "cycle": P.Array(P.Double()), "multiplicative": P.Boolean() }) }], P.Wildcard("A")) errcodeBase = 14050 def __call__(self, state, scope, pos, paramTypes, x, alpha, beta, gamma, theState): if alpha < 0.0 or alpha > 1.0: raise PFARuntimeException("alpha out of range", self.errcodeBase + 0, self.name, pos) if beta < 0.0 or beta > 1.0: raise PFARuntimeException("beta out of range", self.errcodeBase + 1, self.name, pos) if gamma < 0.0 or gamma > 1.0: raise PFARuntimeException("gamma out of range", self.errcodeBase + 2, self.name, pos) level_prev = theState["level"] trend_prev = theState["trend"] cycle_unrotated = theState["cycle"] if len(cycle_unrotated) == 0: raise PFARuntimeException("empty cycle", self.errcodeBase + 3, self.name, pos) cycle_rotated = cycle_unrotated[1:] + [cycle_unrotated[0]] cycle_prev = cycle_rotated[0] if theState["multiplicative"]: level = div(alpha * x, cycle_prev) + (1.0 - alpha) * (level_prev + trend_prev) trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev cycle = div(gamma * x, level) + (1.0 - gamma) * cycle_prev else: level = alpha * (x - cycle_prev) + (1.0 - alpha) * (level_prev + trend_prev) trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev cycle = gamma * (x - level) + (1.0 - gamma) * cycle_prev return dict(theState, level=level, trend=trend, cycle=([cycle] + cycle_rotated[1:]))
class UpdateWindow(UpdateMeanVariance): name = prefix + "updateWindow" sig = Sig([{ "x": P.Double() }, { "w": P.Double() }, { "state": P.Array( P.WildRecord("A", { "x": P.Double(), "w": P.Double(), "count": P.Double() })) }, { "windowSize": P.Int() }], P.Array(P.Wildcard("A"))) errcodeBase = 14020 def _getRecord(self, paramType): return paramType.items def __call__(self, state, scope, pos, paramTypes, x, w, theState, windowSize, level): if windowSize < 2: raise PFARuntimeException("windowSize must be at least 2", self.errcodeBase + 0, self.name, pos) if len(theState) == 0: out = {} paramType = state.parser.getAvroType(paramTypes[2]) for field in self._getRecord(paramType).fields: if field.name == "x": out["x"] = x elif field.name == "w": out["w"] = w elif field.name == "count": out["count"] = w elif field.name == "mean": out["mean"] = x elif field.name == "variance": out["variance"] = 0.0 else: raise PFARuntimeException( "cannot initialize unrecognized fields", self.errcodeBase + 1, self.name, pos) return [out] elif len(theState) >= windowSize: record = theState[-1] splitAt = len(theState) - windowSize + 1 remove, keep = theState[:splitAt], theState[splitAt:] oldx = [xi["x"] for xi in remove] oldw = [-xi["w"] for xi in remove] originalCount = record["count"] count = originalCount + w count2 = count + sum(oldw) if level == 0: return keep + [dict(record, x=x, w=w, count=count2)] else: mean = record["mean"] delta = x - mean shift = div(delta * w, count) mean += shift accumulatedCount = count varianceCorrection = 0.0 for ox, ow in zip(oldx, oldw): accumulatedCount += ow delta2 = ox - mean shift2 = div(delta2 * ow, accumulatedCount) mean += shift2 varianceCorrection += (accumulatedCount - ow) * delta2 * shift2 if level == 1: return keep + [ dict(record, x=x, w=w, count=count2, mean=mean) ] else: varianceTimesCount = record["variance"] * originalCount varianceTimesCount += originalCount * delta * shift varianceTimesCount += varianceCorrection return keep + [ dict(record, x=x, w=w, count=count2, mean=mean, variance=div(varianceTimesCount, count2)) ] else: record = theState[-1] originalCount = record["count"] count = originalCount + w if level == 0: return theState + [dict(record, x=x, w=w, count=count)] else: mean = record["mean"] delta = x - mean shift = div(delta * w, count) mean += shift if level == 1: return theState + [ dict(record, x=x, w=w, count=count, mean=mean) ] else: varianceTimesCount = record["variance"] * originalCount varianceTimesCount += originalCount * delta * shift return theState + [ dict(record, x=x, w=w, count=count, mean=mean, variance=div(varianceTimesCount, count)) ]
class UpdateCovariance(LibFcn): name = prefix + "updateCovariance" sig = Sigs([ Sig([{ "x": P.Array(P.Double()) }, { "w": P.Double() }, { "state": P.WildRecord( "A", { "count": P.Double(), "mean": P.Array(P.Double()), "covariance": P.Array(P.Array(P.Double())) }) }], P.Wildcard("A")), Sig([{ "x": P.Map(P.Double()) }, { "w": P.Double() }, { "state": P.WildRecord( "A", { "count": P.Map(P.Map(P.Double())), "mean": P.Map(P.Double()), "covariance": P.Map(P.Map(P.Double())) }) }], P.Wildcard("A")) ]) errcodeBase = 14010 def __call__(self, state, scope, pos, paramTypes, x, w, theState): size = len(x) if size < 2: raise PFARuntimeException("too few components", self.errcodeBase + 1, self.name, pos) if paramTypes[0]["type"] == "map": oldCount = theState["count"] oldMean = theState["mean"] oldCovariance = theState["covariance"] countKeys = set(oldCount.keys()).union( reduce(lambda a, b: a.union(b), (set(v.keys()) for v in oldCount.values()), set())) covarKeys = set(oldCovariance.keys()).union( reduce(lambda a, b: a.union(b), (set(v.keys()) for v in oldCovariance.values()), set())) keys = set(x.keys()).union(countKeys).union(covarKeys) newCount = {} for i in keys: row = {} for j in keys: old = oldCount.get(i, {}).get(j, 0.0) if (i in x) and (j in x): row[j] = old + w else: row[j] = old newCount[i] = row newMean = {} for i in keys: old = oldMean.get(i, 0.0) if i in x: newMean[i] = old + div((x[i] - old) * w, newCount[i][i]) else: newMean[i] = old newCovariance = {} for i in keys: row = {} for j in keys: oldCov = oldCovariance.get(i, {}).get(j, 0.0) if (i in x) and (j in x): oldC = oldCount.get(i, {}).get(j, 0.0) oldMi = oldMean.get(i, 0.0) oldMj = oldMean.get(j, 0.0) row[j] = div(((oldCov * oldC) + div( (x[i] - oldMi) * (x[j] - oldMj) * w * oldC, newCount[i][j])), newCount[i][j]) else: row[j] = oldCov newCovariance[i] = row return dict(theState, count=newCount, mean=newMean, covariance=newCovariance) else: oldCount = theState["count"] oldMean = theState["mean"] oldCovariance = theState["covariance"] if (size != len(oldMean) or size != len(oldCovariance) or any(size != len(xi) for xi in oldCovariance)): raise PFARuntimeException("unequal length arrays", self.errcodeBase + 2, self.name, pos) newCount = oldCount + w newMean = [ oldm + div((x[i] - oldm) * w, newCount) for i, oldm in enumerate(oldMean) ] newCovariance = [] for i in range(size): row = [] for j in range(size): row.append( div((oldCovariance[i][j] * oldCount) + div( (x[i] - oldMean[i]) * (x[j] - oldMean[j]) * w * oldCount, newCount), newCount)) newCovariance.append(row) return dict(theState, count=newCount, mean=newMean, covariance=newCovariance)