class RandomHistogram(LibFcn):
name = prefix + "histogram"
sig = Sigs([
Sig([{
"distribution": P.Array(P.Double())
}], P.Int()),
Sig([{
"distribution": P.Array(P.WildRecord("A", {"prob": P.Double()}))
}], P.Wildcard("A"))
])
errcodeBase = 34070
def selectIndex(self, rand, distribution, pos):
cumulativeSum = [0.0]
for x in distribution:
if math.isnan(x) or math.isinf(x):
raise PFARuntimeException("distribution must be finite",
self.errcodeBase + 1, self.name, pos)
elif x < 0:
raise PFARuntimeException("distribution must be non-negative",
self.errcodeBase + 2, self.name, pos)
cumulativeSum.append(cumulativeSum[-1] + x)
total = cumulativeSum[-1]
if total == 0.0:
raise PFARuntimeException("distribution must be non-empty",
self.errcodeBase + 0, self.name, pos)
position = rand.uniform(0.0, total)
for i, y in enumerate(cumulativeSum):
if position < y:
return i - 1
def __call__(self, state, scope, pos, paramTypes, distribution):
if isinstance(paramTypes[-1],
dict) and paramTypes[-1].get("type") == "record":
probs = [x["prob"] for x in distribution]
index = self.selectIndex(state.rand, probs, pos)
return distribution[index]
else:
return self.selectIndex(state.rand, distribution, pos)
class Argmin(LibFcn):
name = prefix + "argmin"
sig = Sig([{"m": P.Map(P.Wildcard("A"))}], P.String())
errcodeBase = 26130
def __call__(self, state, scope, pos, paramTypes, m):
if len(m) == 0:
raise PFARuntimeException("empty map", self.errcodeBase + 0,
self.name, pos)
else:
return argLowestN(
m, 1, lambda x, y: compare(
jsonNodeToAvroType(paramTypes[0]).values, x, y) < 0)[0]
class Subseq(LibFcn):
name = prefix + "subseq"
sig = Sig([{
"x": P.Bytes()
}, {
"start": P.Int()
}, {
"end": P.Int()
}], P.Bytes())
errcodeBase = 16010
def __call__(self, state, scope, pos, paramTypes, x, start, end):
return x[start:end]
class FromBase64(LibFcn):
name = prefix + "fromBase64"
sig = Sig([{"s": P.String()}], P.Bytes())
errcodeBase = 16220
def __call__(self, state, scope, pos, paramTypes, s):
try:
if re.match("^[A-Za-z0-9\+/]*=*$", s) is None:
raise TypeError
return base64.b64decode(s)
except TypeError:
raise PFARuntimeException("invalid base64", self.errcodeBase + 0,
self.name, pos)
class RandomLong(LibFcn):
name = prefix + "long"
sig = Sigs([
Sig([], P.Long()),
Sig([{
"long": P.Long()
}, {
"high": P.Long()
}], P.Long())
])
errcodeBase = 34010
def __call__(self, state, scope, pos, paramTypes, *args):
if len(args) == 0:
return state.rand.randint(LONG_MIN_VALUE, LONG_MAX_VALUE)
else:
if args[1] <= args[0]:
raise PFARuntimeException("high must be greater than low",
self.errcodeBase + 0, self.name, pos)
return state.rand.randint(
args[0],
args[1] - 1) # Python's randint has an inclusive upper bound
class Logsumexp(LibFcn):
name = prefix + "logsumexp"
sig = Sig([{"a": P.Array(P.Double())}], P.Double())
errcodeBase = 15480
def __call__(self, state, scope, pos, paramTypes, datum):
if len(datum) == 0:
return float("nan")
else:
dmax = max(datum)
res = 0.0
for v in datum:
res += math.exp(v - dmax)
return math.log(res) + dmax
class BitwiseOr(LibFcn):
name = "|"
sig = Sigs([
Sig([{
"x": P.Int()
}, {
"y": P.Int()
}], P.Int()),
Sig([{
"x": P.Long()
}, {
"y": P.Long()
}], P.Long())
])
errcodeBase = 18260
def genpy(self, paramTypes, args, pos):
return "({0} | {1})".format(*args)
def __call__(self, state, scope, pos, paramTypes, x, y):
return x | y
class FindAll(LibFcn):
name = prefix + "findall"
sig = Sigs([Sig([{"haystack": P.String()}, {"pattern": P.String()}], P.Array(P.String())),
Sig([{"haystack": P.Bytes()}, {"pattern": P.Bytes()}], P.Array(P.Bytes()))])
errcodeBase = 35060
def __call__(self, state, scope, pos, paramTypes, haystack, pattern):
haystack, pattern, to = convert(haystack, pattern, paramTypes[0])
re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos)
found = re.search(0)
region = re.getRegion()
start = region.end[0]
out = []
if found:
while found:
region = re.getRegion()
start = region.end[0]
out.append(to(haystack[region.beg[0]:region.end[0]]))
found = re.search(start)
else:
out = []
re.free()
return out
class Index(LibFcn):
name = prefix + "index"
sig = Sigs([Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Array(P.Wildcard("A"))}], P.Int()),
Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Wildcard("A")}], P.Int()),
Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Fcn([P.Wildcard("A")], P.Boolean())}], P.Int())])
errcodeBase = 15090
def __call__(self, state, scope, pos, paramTypes, haystack, needle):
if isinstance(needle, (list, tuple)):
for start in range(len(haystack) - len(needle) + 1):
if needle == haystack[start:(start + len(needle))]:
return start
return -1
elif callable(needle):
for index, item in enumerate(haystack):
if callfcn(state, scope, needle, [item]):
return index
return -1
else:
try:
return haystack.index(needle)
except ValueError:
return -1
class IsWorkHours(LibFcn):
name = prefix + "isWorkHours"
sig = Sig([{"ts": P.Double()}, {"zone": P.String()}], P.Boolean())
errcodeBase = 40170
def __call__(self, state, scope, pos, paramTypes, ts, zone):
dt = tz(
datetime.datetime.fromtimestamp(
tscheck(ts, self.errcodeBase + 1, self.name, pos), UTC()),
zone, self.errcodeBase + 0, self.name, pos)
day = dt.weekday()
hour = dt.hour
return (day != 5 or day != 6) and (hour >= 9 and hour < 17)
class SymDiff(LibFcn):
name = prefix + "symdiff"
sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("A"))}], P.Array(P.Wildcard("A")))
errcodeBase = 15540
def __call__(self, state, scope, pos, paramTypes, a, b):
out = []
for ai in a:
if ai not in b:
out.append(ai)
for bi in b:
if bi not in a:
out.append(bi)
return out
class Rotate(LibFcn):
name = prefix + "rotate"
sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"steps": P.Int()}], P.Array(P.Wildcard("A")))
errcodeBase = 15190
def __call__(self, state, scope, pos, paramTypes, a, steps):
if steps < 0:
raise PFARuntimeException("steps out of range", self.errcodeBase + 0, self.name, pos)
elif steps == 0 or len(a) == 0:
return a
else:
index = steps % len(a)
left, right = a[:index], a[index:]
return right + left
class Contains(LibFcn):
name = prefix + "contains"
sig = Sigs([
Sig([{
"haystack": P.String()
}, {
"pattern": P.String()
}], P.Boolean()),
Sig([{
"haystack": P.Bytes()
}, {
"pattern": P.Bytes()
}], P.Boolean())
])
errcodeBase = 35010
def __call__(self, state, scope, pos, paramTypes, haystack, pattern):
haystack, pattern, to = convert(haystack, pattern, paramTypes[0])
re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos)
out = re.search(0)
re.free()
return out
class Lnsum(LibFcn):
name = prefix + "lnsum"
sig = Sig([{"a": P.Array(P.Double())}], P.Double())
errcodeBase = 15420
def ln(self, x):
if x < 0.0:
return float("nan")
elif x == 0.0:
return float("-inf")
else:
return math.log(x)
def __call__(self, state, scope, pos, paramTypes, a):
return sum([self.ln(x) for x in a])
class Intersection(LibFcn):
name = prefix + "intersection"
sig = Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("A"))}], P.Array(P.Wildcard("A")))
errcodeBase = 15520
def __call__(self, state, scope, pos, paramTypes, a, b):
out = []
for ai in a:
if ai in b:
out.append(ai)
for bi in b:
if bi in a:
out.append(bi)
return out
class RIndex(LibFcn):
name = prefix + "rindex"
sig = Sigs([Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Array(P.Wildcard("A"))}], P.Int()),
Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Wildcard("A")}], P.Int()),
Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Fcn([P.Wildcard("A")], P.Boolean())}], P.Int())])
errcodeBase = 15100
def __call__(self, state, scope, pos, paramTypes, haystack, needle):
if isinstance(needle, (list, tuple)):
for start in range(len(haystack) - len(needle), -1, -1):
if needle == haystack[start:(start + len(needle))]:
return start
return -1
elif callable(needle):
for index, item in enumerate(reversed(haystack)):
if callfcn(state, scope, needle, [item]):
return len(haystack) - 1 - index
return -1
else:
for index in range(len(haystack) - 1, -1, -1):
if needle == haystack[index]:
return index
return -1
class Transpose(LibFcn):
name = prefix + "transpose"
sig = Sigs([
Sig([{
"x": P.Array(P.Array(P.Double()))
}], P.Array(P.Array(P.Double()))),
Sig([{
"x": P.Map(P.Map(P.Double()))
}], P.Map(P.Map(P.Double())))
])
errcodeBase = 24060
def __call__(self, state, scope, pos, paramTypes, x):
if isinstance(x, (list, tuple)) and all(
isinstance(xi, (list, tuple)) for xi in x):
rows = len(x)
if rows < 1:
raise PFARuntimeException("too few rows/cols",
self.errcodeBase + 0, self.name, pos)
cols = len(x[0])
if cols < 1:
raise PFARuntimeException("too few rows/cols",
self.errcodeBase + 0, self.name, pos)
if raggedArray(x):
raise PFARuntimeException("ragged columns",
self.errcodeBase + 1, self.name, pos)
return [[x[r][c] for r in range(rows)] for c in range(cols)]
elif isinstance(x, dict) and all(
isinstance(x[i], dict) for i in list(x.keys())):
rows = rowKeys(x)
cols = colKeys(x)
if len(rows) < 1 or len(cols) < 1:
raise PFARuntimeException("too few rows/cols",
self.errcodeBase + 0, self.name, pos)
if raggedMap(x):
raise PFARuntimeException("ragged columns",
self.errcodeBase + 1, self.name, pos)
return dict((c, dict((r, x[r][c]) for r in rows)) for c in cols)
class Decoder(LibFcn):
sig = Sig([{"x": P.Bytes()}], P.String())
codec = None
def __call__(self, state, scope, pos, paramTypes, x):
# Hack for Python 3: return without doing anything
if sys.version_info[0] == 3 and isinstance(x, str):
return x
try:
return codecs.decode(x, self.codec, "strict")
except UnicodeDecodeError as err:
raise PFARuntimeException("invalid bytes", self.errcodeBase + 0,
self.name, pos)
class Permutations(LibFcn):
name = prefix + "permutations"
sig = Sig([{"a": P.Array(P.Wildcard("A"))}], P.Array(P.Array(P.Wildcard("A"))))
errcodeBase = 15790
def __call__(self, state, scope, pos, paramTypes, a):
out = []
i = 0
for permutation in itertools.permutations(a):
i += 1
if i % 1000 == 0:
state.checkTime()
out.append(list(permutation))
return out
class GaussianSimilarity(LibFcn):
name = prefix + "gaussianSimilarity"
sig = Sig([{
"x": P.Double()
}, {
"y": P.Double()
}, {
"sigma": P.Double()
}], P.Double())
errcodeBase = 28020
def __call__(self, state, scope, pos, paramTypes, x, y, sigma):
return math.exp(div(-math.log(2) * (x - y)**2, sigma**2))
class CastAvro(LibFcn):
name = prefix + "avro"
sig = Sig([{"x": P.Wildcard("A")}], P.Bytes())
errcodeBase = 17110
def __call__(self, state, scope, pos, paramTypes, x):
schema = avro.schema.Parse(json.dumps(paramTypes[0]))
x = untagUnion(x, paramTypes[0])
bytes = io.BytesIO()
writer = DatumWriter(schema)
writer.write(x, BinaryEncoder(bytes))
bytes.flush()
return bytes.getvalue()
class FanoutLong(LibFcn):
name = prefix + "fanoutLong"
sig = Sigs([
Sig([{
"x": P.WildEnum("A")
}], P.Array(P.Long())),
Sig([{
"x": P.String()
}, {
"dictionary": P.Array(P.String())
}, {
"outOfRange": P.Boolean()
}], P.Array(P.Long())),
Sig([{
"x": P.Int()
}, {
"minimum": P.Int()
}, {
"maximum": P.Int()
}, {
"outOfRange": P.Boolean()
}], P.Array(P.Long()))
])
errcodeBase = 17080
def __call__(self, state, scope, pos, paramTypes, x, *args):
if len(args) == 0:
return [
1 if y else 0 for y in fanoutEnum(x, paramTypes[0]["symbols"])
]
elif len(args) == 2:
if len(args[0]) != len(set(args[0])):
raise PFARuntimeException("non-distinct values in dictionary",
self.errcodeBase + 0, self.name, pos)
return [1 if y else 0 for y in fanoutString(x, args[0], args[1])]
elif len(args) == 3:
return [
1 if y else 0 for y in fanoutInt(x, args[0], args[1], args[2])
]
class UpdateEWMA(LibFcn):
name = prefix + "updateEWMA"
sig = Sig([{
"x": P.Double()
}, {
"alpha": P.Double()
}, {
"state": P.WildRecord("A", {"mean": P.Double()})
}], P.Wildcard("A"))
errcodeBase = 14030
def _getRecord(self, paramType):
if isinstance(paramType, AvroUnion):
for t in paramType.types:
if not isinstance(t, AvroNull):
return t
else:
return paramType
def genpy(self, paramTypes, args, pos):
hasVariance = False
for x in self._getRecord(paramTypes[2]).fields:
if x.name == "variance":
if not x.avroType.accepts(AvroDouble()):
raise PFASemanticException(
self.name +
" is being given a state record type in which the \"variance\" field is not a double: "
+ str(x.avroType), None)
hasVariance = True
return "self.f[{0}]({1}, {2})".format(
repr(self.name), ", ".join(
["state", "scope",
repr(pos), repr(paramTypes)] + args), hasVariance)
def __call__(self, state, scope, pos, paramTypes, x, alpha, theState,
hasVariance):
if alpha < 0.0 or alpha > 1.0:
raise PFARuntimeException("alpha out of range",
self.errcodeBase + 0, self.name, pos)
mean = theState["mean"]
diff = x - mean
incr = alpha * diff
if hasVariance:
variance = theState["variance"]
return dict(theState,
mean=(mean + incr),
variance=((1.0 - alpha) * (variance + diff * incr)))
else:
return dict(theState, mean=(mean + incr))
class ReplaceLast(LibFcn):
name = prefix + "replacelast"
sig = Sigs([
Sig([{
"haystack": P.String()
}, {
"pattern": P.String()
}, {
"replacement": P.String()
}], P.String()),
Sig([{
"haystack": P.Bytes()
}, {
"pattern": P.Bytes()
}, {
"replacement": P.Bytes()
}], P.Bytes())
])
errcodeBase = 35120
def __call__(self, state, scope, pos, paramTypes, haystack, pattern,
replacement):
haystack, pattern, to = convert(haystack, pattern, paramTypes[0])
re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos)
found = re.search(0)
region = re.getRegion()
start = 0
if found:
while found:
region = re.getRegion()
start = region.end[0]
found = re.search(start)
out = to(haystack[:region.beg[0]]) + replacement + to(
haystack[region.end[0]:])
else:
out = to(haystack)
re.free()
return out
class LinearMissing(LibFcn):
name = prefix + "linearMissing"
sig = Sigs([
Sig([{
"x": P.Double()
}, {
"table":
P.Array(P.WildRecord("R", {
"x": P.Double(),
"to": P.Double()
}))
}], P.Union([P.Null(), P.Double()])),
Sig([{
"x": P.Double()
}, {
"table":
P.Array(
P.WildRecord("R", {
"x": P.Double(),
"to": P.Array(P.Double())
}))
}], P.Union([P.Null(), P.Array(P.Double())]))
])
errcodeBase = 22040
def __call__(self, state, scope, pos, paramTypes, datum, table):
one, two, between = Linear.closest(datum, table, self.errcodeBase + 0,
self.name, pos)
if not between and one["x"] != datum:
return None
elif isinstance([x for x in paramTypes[-1] if x != "null"][0],
dict) and [x for x in paramTypes[-1] if x != "null"
][0].get("type") == "array":
return Linear.interpolateMulti(datum, one, two,
self.errcodeBase + 1, self.name,
pos)
else:
return Linear.interpolateSingle(datum, one, two)
class Scale(LibFcn):
name = prefix + "scale"
sig = Sigs([
Sig([{
"x": P.Array(P.Double())
}, {
"alpha": P.Double()
}], P.Array(P.Double())),
Sig([{
"x": P.Array(P.Array(P.Double()))
}, {
"alpha": P.Double()
}], P.Array(P.Array(P.Double()))),
Sig([{
"x": P.Map(P.Double())
}, {
"alpha": P.Double()
}], P.Map(P.Double())),
Sig([{
"x": P.Map(P.Map(P.Double()))
}, {
"alpha": P.Double()
}], P.Map(P.Map(P.Double())))
])
errcodeBase = 24010
def __call__(self, state, scope, pos, paramTypes, x, alpha):
if isinstance(x, (list, tuple)) and all(
isinstance(xi, (list, tuple)) for xi in x):
return [[xj * alpha for xj in xi] for xi in x]
elif isinstance(x, (list, tuple)):
return [xi * alpha for xi in x]
elif isinstance(x, dict) and all(isinstance(x[i], dict) for i in x):
return dict((i, dict((j, xj * alpha)
for j, xj in list(xi.items())))
for i, xi in list(x.items()))
else:
return dict((i, xi * alpha) for i, xi in list(x.items()))
class SimpleEuclidean(LibFcn):
name = prefix + "simpleEuclidean"
sig = Sig([{
"x": P.Array(P.Double())
}, {
"y": P.Array(P.Double())
}], P.Double())
errcodeBase = 28000
def __call__(self, state, scope, pos, paramTypes, x, y):
if len(x) != len(y):
raise PFARuntimeException("dimensions of vectors do not match",
self.errcodeBase + 0, self.name, pos)
return math.sqrt(sum(((xi - yi)**2 for xi, yi in zip(x, y))))
class UpdateHoltWinters(LibFcn):
name = prefix + "updateHoltWinters"
sig = Sig([{"x": P.Double()}, {"alpha": P.Double()}, {"beta": P.Double()}, {"state": P.WildRecord("A", {"level": P.Double(), "trend": P.Double()})}], P.Wildcard("A"))
errcodeBase = 14040
def __call__(self, state, scope, pos, paramTypes, x, alpha, beta, theState):
if alpha < 0.0 or alpha > 1.0:
raise PFARuntimeException("alpha out of range", self.errcodeBase + 0, self.name, pos)
if beta < 0.0 or beta > 1.0:
raise PFARuntimeException("beta out of range", self.errcodeBase + 1, self.name, pos)
level_prev = theState["level"]
trend_prev = theState["trend"]
level = alpha * x + (1.0 - alpha) * (level_prev + trend_prev)
trend = beta * (level - level_prev) + (1.0 - beta) * trend_prev
return dict(theState, level=level, trend=trend)
class ReplaceAll(LibFcn):
name = prefix + "replaceall"
sig = Sig([{
"s": P.String()
}, {
"original": P.String()
}, {
"replacement": P.String()
}], P.String())
errcodeBase = 39200
def __call__(self, state, scope, pos, paramTypes, s, original,
replacement):
return s.replace(original, replacement)
class ToDouble(LibFcn):
name = prefix + "double"
sig = Sigs([
Sig([{
"x": P.Int()
}], P.Double()),
Sig([{
"x": P.Long()
}], P.Double()),
Sig([{
"x": P.Float()
}], P.Double()),
Sig([{
"x": P.Double()
}], P.Double())
])
errcodeBase = 17050
def genpy(self, paramTypes, args, pos):
return "float({0})".format(*args)
def __call__(self, state, scope, pos, paramTypes, x):
return float(x)
