class RandomHistogram(LibFcn): name = prefix + "histogram" sig = Sigs([ Sig([{ "distribution": P.Array(P.Double()) }], P.Int()), Sig([{ "distribution": P.Array(P.WildRecord("A", {"prob": P.Double()})) }], P.Wildcard("A")) ]) errcodeBase = 34070 def selectIndex(self, rand, distribution, pos): cumulativeSum = [0.0] for x in distribution: if math.isnan(x) or math.isinf(x): raise PFARuntimeException("distribution must be finite", self.errcodeBase + 1, self.name, pos) elif x < 0: raise PFARuntimeException("distribution must be non-negative", self.errcodeBase + 2, self.name, pos) cumulativeSum.append(cumulativeSum[-1] + x) total = cumulativeSum[-1] if total == 0.0: raise PFARuntimeException("distribution must be non-empty", self.errcodeBase + 0, self.name, pos) position = rand.uniform(0.0, total) for i, y in enumerate(cumulativeSum): if position < y: return i - 1 def __call__(self, state, scope, pos, paramTypes, distribution): if isinstance(paramTypes[-1], dict) and paramTypes[-1].get("type") == "record": probs = [x["prob"] for x in distribution] index = self.selectIndex(state.rand, probs, pos) return distribution[index] else: return self.selectIndex(state.rand, distribution, pos)
class Argmin(LibFcn): name = prefix + "argmin" sig = Sig([{"m": P.Map(P.Wildcard("A"))}], P.String()) errcodeBase = 26130 def __call__(self, state, scope, pos, paramTypes, m): if len(m) == 0: raise PFARuntimeException("empty map", self.errcodeBase + 0, self.name, pos) else: return argLowestN( m, 1, lambda x, y: compare( jsonNodeToAvroType(paramTypes[0]).values, x, y) < 0)[0]
class Subseq(LibFcn): name = prefix + "subseq" sig = Sig([{ "x": P.Bytes() }, { "start": P.Int() }, { "end": P.Int() }], P.Bytes()) errcodeBase = 16010 def __call__(self, state, scope, pos, paramTypes, x, start, end): return x[start:end]
class FromBase64(LibFcn): name = prefix + "fromBase64" sig = Sig([{"s": P.String()}], P.Bytes()) errcodeBase = 16220 def __call__(self, state, scope, pos, paramTypes, s): try: if re.match("^[A-Za-z0-9\+/]*=*$", s) is None: raise TypeError return base64.b64decode(s) except TypeError: raise PFARuntimeException("invalid base64", self.errcodeBase + 0, self.name, pos)
class RandomLong(LibFcn): name = prefix + "long" sig = Sigs([ Sig([], P.Long()), Sig([{ "long": P.Long() }, { "high": P.Long() }], P.Long()) ]) errcodeBase = 34010 def __call__(self, state, scope, pos, paramTypes, *args): if len(args) == 0: return state.rand.randint(LONG_MIN_VALUE, LONG_MAX_VALUE) else: if args[1] <= args[0]: raise PFARuntimeException("high must be greater than low", self.errcodeBase + 0, self.name, pos) return state.rand.randint( args[0], args[1] - 1) # Python's randint has an inclusive upper bound
class Logsumexp(LibFcn): name = prefix + "logsumexp" sig = Sig([{"a": P.Array(P.Double())}], P.Double()) errcodeBase = 15480 def __call__(self, state, scope, pos, paramTypes, datum): if len(datum) == 0: return float("nan") else: dmax = max(datum) res = 0.0 for v in datum: res += math.exp(v - dmax) return math.log(res) + dmax
class BitwiseOr(LibFcn): name = "|" sig = Sigs([ Sig([{ "x": P.Int() }, { "y": P.Int() }], P.Int()), Sig([{ "x": P.Long() }, { "y": P.Long() }], P.Long()) ]) errcodeBase = 18260 def genpy(self, paramTypes, args, pos): return "({0} | {1})".format(*args) def __call__(self, state, scope, pos, paramTypes, x, y): return x | y
class FindAll(LibFcn): name = prefix + "findall" sig = Sigs([Sig([{"haystack": P.String()}, {"pattern": P.String()}], P.Array(P.String())), Sig([{"haystack": P.Bytes()}, {"pattern": P.Bytes()}], P.Array(P.Bytes()))]) errcodeBase = 35060 def __call__(self, state, scope, pos, paramTypes, haystack, pattern): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) found = re.search(0) region = re.getRegion() start = region.end[0] out = [] if found: while found: region = re.getRegion() start = region.end[0] out.append(to(haystack[region.beg[0]:region.end[0]])) found = re.search(start) else: out = [] re.free() return out
class Index(LibFcn): name = prefix + "index" sig = Sigs([Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Array(P.Wildcard("A"))}], P.Int()), Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Wildcard("A")}], P.Int()), Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Fcn([P.Wildcard("A")], P.Boolean())}], P.Int())]) errcodeBase = 15090 def __call__(self, state, scope, pos, paramTypes, haystack, needle): if isinstance(needle, (list, tuple)): for start in range(len(haystack) - len(needle) + 1): if needle == haystack[start:(start + len(needle))]: return start return -1 elif callable(needle): for index, item in enumerate(haystack): if callfcn(state, scope, needle, [item]): return index return -1 else: try: return haystack.index(needle) except ValueError: return -1
class IsWorkHours(LibFcn): name = prefix + "isWorkHours" sig = Sig([{"ts": P.Double()}, {"zone": P.String()}], P.Boolean()) errcodeBase = 40170 def __call__(self, state, scope, pos, paramTypes, ts, zone): dt = tz( datetime.datetime.fromtimestamp( tscheck(ts, self.errcodeBase + 1, self.name, pos), UTC()), zone, self.errcodeBase + 0, self.name, pos) day = dt.weekday() hour = dt.hour return (day != 5 or day != 6) and (hour >= 9 and hour < 17)
class SymDiff(LibFcn): name = prefix + "symdiff" sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("A"))}], P.Array(P.Wildcard("A"))) errcodeBase = 15540 def __call__(self, state, scope, pos, paramTypes, a, b): out = [] for ai in a: if ai not in b: out.append(ai) for bi in b: if bi not in a: out.append(bi) return out
class Rotate(LibFcn): name = prefix + "rotate" sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"steps": P.Int()}], P.Array(P.Wildcard("A"))) errcodeBase = 15190 def __call__(self, state, scope, pos, paramTypes, a, steps): if steps < 0: raise PFARuntimeException("steps out of range", self.errcodeBase + 0, self.name, pos) elif steps == 0 or len(a) == 0: return a else: index = steps % len(a) left, right = a[:index], a[index:] return right + left
class Contains(LibFcn): name = prefix + "contains" sig = Sigs([ Sig([{ "haystack": P.String() }, { "pattern": P.String() }], P.Boolean()), Sig([{ "haystack": P.Bytes() }, { "pattern": P.Bytes() }], P.Boolean()) ]) errcodeBase = 35010 def __call__(self, state, scope, pos, paramTypes, haystack, pattern): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) out = re.search(0) re.free() return out
class Lnsum(LibFcn): name = prefix + "lnsum" sig = Sig([{"a": P.Array(P.Double())}], P.Double()) errcodeBase = 15420 def ln(self, x): if x < 0.0: return float("nan") elif x == 0.0: return float("-inf") else: return math.log(x) def __call__(self, state, scope, pos, paramTypes, a): return sum([self.ln(x) for x in a])
class Intersection(LibFcn): name = prefix + "intersection" sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("A"))}], P.Array(P.Wildcard("A"))) errcodeBase = 15520 def __call__(self, state, scope, pos, paramTypes, a, b): out = [] for ai in a: if ai in b: out.append(ai) for bi in b: if bi in a: out.append(bi) return out
class RIndex(LibFcn): name = prefix + "rindex" sig = Sigs([Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Array(P.Wildcard("A"))}], P.Int()), Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Wildcard("A")}], P.Int()), Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Fcn([P.Wildcard("A")], P.Boolean())}], P.Int())]) errcodeBase = 15100 def __call__(self, state, scope, pos, paramTypes, haystack, needle): if isinstance(needle, (list, tuple)): for start in range(len(haystack) - len(needle), -1, -1): if needle == haystack[start:(start + len(needle))]: return start return -1 elif callable(needle): for index, item in enumerate(reversed(haystack)): if callfcn(state, scope, needle, [item]): return len(haystack) - 1 - index return -1 else: for index in range(len(haystack) - 1, -1, -1): if needle == haystack[index]: return index return -1
class Transpose(LibFcn): name = prefix + "transpose" sig = Sigs([ Sig([{ "x": P.Array(P.Array(P.Double())) }], P.Array(P.Array(P.Double()))), Sig([{ "x": P.Map(P.Map(P.Double())) }], P.Map(P.Map(P.Double()))) ]) errcodeBase = 24060 def __call__(self, state, scope, pos, paramTypes, x): if isinstance(x, (list, tuple)) and all( isinstance(xi, (list, tuple)) for xi in x): rows = len(x) if rows < 1: raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) cols = len(x[0]) if cols < 1: raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) if raggedArray(x): raise PFARuntimeException("ragged columns", self.errcodeBase + 1, self.name, pos) return [[x[r][c] for r in range(rows)] for c in range(cols)] elif isinstance(x, dict) and all( isinstance(x[i], dict) for i in list(x.keys())): rows = rowKeys(x) cols = colKeys(x) if len(rows) < 1 or len(cols) < 1: raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) if raggedMap(x): raise PFARuntimeException("ragged columns", self.errcodeBase + 1, self.name, pos) return dict((c, dict((r, x[r][c]) for r in rows)) for c in cols)
class Decoder(LibFcn): sig = Sig([{"x": P.Bytes()}], P.String()) codec = None def __call__(self, state, scope, pos, paramTypes, x): # Hack for Python 3: return without doing anything if sys.version_info[0] == 3 and isinstance(x, str): return x try: return codecs.decode(x, self.codec, "strict") except UnicodeDecodeError as err: raise PFARuntimeException("invalid bytes", self.errcodeBase + 0, self.name, pos)
class Permutations(LibFcn): name = prefix + "permutations" sig = Sig([{"a": P.Array(P.Wildcard("A"))}], P.Array(P.Array(P.Wildcard("A")))) errcodeBase = 15790 def __call__(self, state, scope, pos, paramTypes, a): out = [] i = 0 for permutation in itertools.permutations(a): i += 1 if i % 1000 == 0: state.checkTime() out.append(list(permutation)) return out
class GaussianSimilarity(LibFcn): name = prefix + "gaussianSimilarity" sig = Sig([{ "x": P.Double() }, { "y": P.Double() }, { "sigma": P.Double() }], P.Double()) errcodeBase = 28020 def __call__(self, state, scope, pos, paramTypes, x, y, sigma): return math.exp(div(-math.log(2) * (x - y)**2, sigma**2))
class CastAvro(LibFcn): name = prefix + "avro" sig = Sig([{"x": P.Wildcard("A")}], P.Bytes()) errcodeBase = 17110 def __call__(self, state, scope, pos, paramTypes, x): schema = avro.schema.Parse(json.dumps(paramTypes[0])) x = untagUnion(x, paramTypes[0]) bytes = io.BytesIO() writer = DatumWriter(schema) writer.write(x, BinaryEncoder(bytes)) bytes.flush() return bytes.getvalue()
class FanoutLong(LibFcn): name = prefix + "fanoutLong" sig = Sigs([ Sig([{ "x": P.WildEnum("A") }], P.Array(P.Long())), Sig([{ "x": P.String() }, { "dictionary": P.Array(P.String()) }, { "outOfRange": P.Boolean() }], P.Array(P.Long())), Sig([{ "x": P.Int() }, { "minimum": P.Int() }, { "maximum": P.Int() }, { "outOfRange": P.Boolean() }], P.Array(P.Long())) ]) errcodeBase = 17080 def __call__(self, state, scope, pos, paramTypes, x, *args): if len(args) == 0: return [ 1 if y else 0 for y in fanoutEnum(x, paramTypes[0]["symbols"]) ] elif len(args) == 2: if len(args[0]) != len(set(args[0])): raise PFARuntimeException("non-distinct values in dictionary", self.errcodeBase + 0, self.name, pos) return [1 if y else 0 for y in fanoutString(x, args[0], args[1])] elif len(args) == 3: return [ 1 if y else 0 for y in fanoutInt(x, args[0], args[1], args[2]) ]
class UpdateEWMA(LibFcn): name = prefix + "updateEWMA" sig = Sig([{ "x": P.Double() }, { "alpha": P.Double() }, { "state": P.WildRecord("A", {"mean": P.Double()}) }], P.Wildcard("A")) errcodeBase = 14030 def _getRecord(self, paramType): if isinstance(paramType, AvroUnion): for t in paramType.types: if not isinstance(t, AvroNull): return t else: return paramType def genpy(self, paramTypes, args, pos): hasVariance = False for x in self._getRecord(paramTypes[2]).fields: if x.name == "variance": if not x.avroType.accepts(AvroDouble()): raise PFASemanticException( self.name + " is being given a state record type in which the \"variance\" field is not a double: " + str(x.avroType), None) hasVariance = True return "self.f[{0}]({1}, {2})".format( repr(self.name), ", ".join( ["state", "scope", repr(pos), repr(paramTypes)] + args), hasVariance) def __call__(self, state, scope, pos, paramTypes, x, alpha, theState, hasVariance): if alpha < 0.0 or alpha > 1.0: raise PFARuntimeException("alpha out of range", self.errcodeBase + 0, self.name, pos) mean = theState["mean"] diff = x - mean incr = alpha * diff if hasVariance: variance = theState["variance"] return dict(theState, mean=(mean + incr), variance=((1.0 - alpha) * (variance + diff * incr))) else: return dict(theState, mean=(mean + incr))
class ReplaceLast(LibFcn): name = prefix + "replacelast" sig = Sigs([ Sig([{ "haystack": P.String() }, { "pattern": P.String() }, { "replacement": P.String() }], P.String()), Sig([{ "haystack": P.Bytes() }, { "pattern": P.Bytes() }, { "replacement": P.Bytes() }], P.Bytes()) ]) errcodeBase = 35120 def __call__(self, state, scope, pos, paramTypes, haystack, pattern, replacement): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) found = re.search(0) region = re.getRegion() start = 0 if found: while found: region = re.getRegion() start = region.end[0] found = re.search(start) out = to(haystack[:region.beg[0]]) + replacement + to( haystack[region.end[0]:]) else: out = to(haystack) re.free() return out
class LinearMissing(LibFcn): name = prefix + "linearMissing" sig = Sigs([ Sig([{ "x": P.Double() }, { "table": P.Array(P.WildRecord("R", { "x": P.Double(), "to": P.Double() })) }], P.Union([P.Null(), P.Double()])), Sig([{ "x": P.Double() }, { "table": P.Array( P.WildRecord("R", { "x": P.Double(), "to": P.Array(P.Double()) })) }], P.Union([P.Null(), P.Array(P.Double())])) ]) errcodeBase = 22040 def __call__(self, state, scope, pos, paramTypes, datum, table): one, two, between = Linear.closest(datum, table, self.errcodeBase + 0, self.name, pos) if not between and one["x"] != datum: return None elif isinstance([x for x in paramTypes[-1] if x != "null"][0], dict) and [x for x in paramTypes[-1] if x != "null" ][0].get("type") == "array": return Linear.interpolateMulti(datum, one, two, self.errcodeBase + 1, self.name, pos) else: return Linear.interpolateSingle(datum, one, two)
class Scale(LibFcn): name = prefix + "scale" sig = Sigs([ Sig([{ "x": P.Array(P.Double()) }, { "alpha": P.Double() }], P.Array(P.Double())), Sig([{ "x": P.Array(P.Array(P.Double())) }, { "alpha": P.Double() }], P.Array(P.Array(P.Double()))), Sig([{ "x": P.Map(P.Double()) }, { "alpha": P.Double() }], P.Map(P.Double())), Sig([{ "x": P.Map(P.Map(P.Double())) }, { "alpha": P.Double() }], P.Map(P.Map(P.Double()))) ]) errcodeBase = 24010 def __call__(self, state, scope, pos, paramTypes, x, alpha): if isinstance(x, (list, tuple)) and all( isinstance(xi, (list, tuple)) for xi in x): return [[xj * alpha for xj in xi] for xi in x] elif isinstance(x, (list, tuple)): return [xi * alpha for xi in x] elif isinstance(x, dict) and all(isinstance(x[i], dict) for i in x): return dict((i, dict((j, xj * alpha) for j, xj in list(xi.items()))) for i, xi in list(x.items())) else: return dict((i, xi * alpha) for i, xi in list(x.items()))
class SimpleEuclidean(LibFcn): name = prefix + "simpleEuclidean" sig = Sig([{ "x": P.Array(P.Double()) }, { "y": P.Array(P.Double()) }], P.Double()) errcodeBase = 28000 def __call__(self, state, scope, pos, paramTypes, x, y): if len(x) != len(y): raise PFARuntimeException("dimensions of vectors do not match", self.errcodeBase + 0, self.name, pos) return math.sqrt(sum(((xi - yi)**2 for xi, yi in zip(x, y))))
class UpdateHoltWinters(LibFcn): name = prefix + "updateHoltWinters" sig = Sig([{"x": P.Double()}, {"alpha": P.Double()}, {"beta": P.Double()}, {"state": P.WildRecord("A", {"level": P.Double(), "trend": P.Double()})}], P.Wildcard("A")) errcodeBase = 14040 def __call__(self, state, scope, pos, paramTypes, x, alpha, beta, theState): if alpha < 0.0 or alpha > 1.0: raise PFARuntimeException("alpha out of range", self.errcodeBase + 0, self.name, pos) if beta < 0.0 or beta > 1.0: raise PFARuntimeException("beta out of range", self.errcodeBase + 1, self.name, pos) level_prev = theState["level"] trend_prev = theState["trend"] level = alpha * x + (1.0 - alpha) * (level_prev + trend_prev) trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev return dict(theState, level=level, trend=trend)
class ReplaceAll(LibFcn): name = prefix + "replaceall" sig = Sig([{ "s": P.String() }, { "original": P.String() }, { "replacement": P.String() }], P.String()) errcodeBase = 39200 def __call__(self, state, scope, pos, paramTypes, s, original, replacement): return s.replace(original, replacement)
class ToDouble(LibFcn): name = prefix + "double" sig = Sigs([ Sig([{ "x": P.Int() }], P.Double()), Sig([{ "x": P.Long() }], P.Double()), Sig([{ "x": P.Float() }], P.Double()), Sig([{ "x": P.Double() }], P.Double()) ]) errcodeBase = 17050 def genpy(self, paramTypes, args, pos): return "float({0})".format(*args) def __call__(self, state, scope, pos, paramTypes, x): return float(x)