class SquaredEuclidean(MetricWithMissingValues): name = prefix + "squaredEuclidean" sig = Sigs([ Sig([{ "similarity": P.Fcn([P.Wildcard("A"), P.Wildcard("B")], P.Double()) }, { "x": P.Array(P.Union([P.Null(), P.Wildcard("A")])) }, { "y": P.Array(P.Union([P.Null(), P.Wildcard("B")])) }], P.Double()), Sig([{ "similarity": P.Fcn([P.Wildcard("A"), P.Wildcard("B")], P.Double()) }, { "x": P.Array(P.Union([P.Null(), P.Wildcard("A")])) }, { "y": P.Array(P.Union([P.Null(), P.Wildcard("B")])) }, { "missingWeight": P.Array(P.Double()) }], P.Double()) ]) errcodeBase = 28040 def increment(self, tally, x): return tally + x**2 def finalize(self, x): return x
class Bernoulli(LibFcn): name = prefix + "bernoulli" sig = Sigs([ Sig([{ "datum": P.Array(P.String()) }, { "classModel": P.Map(P.Double()) }], P.Double()), Sig([{ "datum": P.Array(P.String()) }, { "classModel": P.WildRecord("C", {"values": P.Map(P.Double())}) }], P.Double()) ]) errcodeBase = 10020 def __call__(self, state, scope, pos, paramTypes, datum, classModel): if paramTypes[1]["type"] == "record": classModel = classModel["values"] ll = 0.0 for v in classModel.values(): if (v <= 0.0) or (v >= 1.0): raise PFARuntimeException( "probability in classModel cannot be less than 0 or greater than 1", self.errcodeBase + 0, self.name, pos) ll += math.log(1.0 - v) for item in datum: p = classModel.get(item, None) if p is not None: ll += math.log(p) - math.log(1.0 - p) return ll
class Gaussian(LibFcn): name = prefix + "gaussian" sig = Sigs([ Sig([{ "datum": P.Array(P.Double()) }, { "classModel": P.Array( P.WildRecord("C", { "mean": P.Double(), "variance": P.Double() })) }], P.Double()), Sig([{ "datum": P.Map(P.Double()) }, { "classModel": P.Map( P.WildRecord("C", { "mean": P.Double(), "variance": P.Double() })) }], P.Double()) ]) errcodeBase = 10000 def __call__(self, state, scope, pos, paramTypes, datum, classModel): ll = 0.0 if isinstance(datum, list) or isinstance(datum, tuple): if len(datum) != len(classModel): raise PFARuntimeException("datum and classModel misaligned", self.errcodeBase + 0, self.name, pos) for i, x in enumerate(datum): mu = classModel[i]["mean"] vari = classModel[i]["variance"] if vari <= 0.0: raise PFARuntimeException( "variance less than or equal to zero", self.errcodeBase + 1, self.name, pos) ll += -0.5 * math.log(2. * math.pi * vari) ll += -0.5 * ((x - mu)**2 / vari) return ll else: datumkeys = list(datum.keys()) modelkeys = list(classModel.keys()) if set(datumkeys) != set(modelkeys): raise PFARuntimeException("datum and classModel misaligned", self.errcodeBase + 0, self.name, pos) for feature in datumkeys: x = datum[feature] mu = classModel[feature]["mean"] vari = classModel[feature]["variance"] if vari <= 0.0: raise PFARuntimeException( "variance less than or equal to zero", self.errcodeBase + 1, self.name, pos) ll += -0.5 * math.log(2. * math.pi * vari) ll += -0.5 * ((x - mu)**2 / vari) return ll
class DefaultOnNonNum(LibFcn): name = prefix + "defaultOnNonNum" sig = Sigs([Sig([{"x": P.Float()}, {"default": P.Float()}], P.Float()), Sig([{"x": P.Double()}, {"default": P.Double()}], P.Double())]) errcodeBase = 21060 def __call__(self, state, scope, pos, paramTypes, x, default): if math.isnan(x) or math.isinf(x): return default else: return x
class Linear(LibFcn): name = prefix + "linear" sig = Sigs([Sig([{"datum": P.Array(P.Double())}, {"model": P.WildRecord("M", {"coeff": P.Array(P.Double()), "const": P.Double()})}], P.Double()), Sig([{"datum": P.Array(P.Double())}, {"model": P.WildRecord("M", {"coeff": P.Array(P.Array(P.Double())), "const": P.Array(P.Double())})}], P.Array(P.Double())), Sig([{"datum": P.Map(P.Double())}, {"model": P.WildRecord("M", {"coeff": P.Map(P.Double()), "const": P.Double()})}], P.Double()), Sig([{"datum": P.Map(P.Double())}, {"model": P.WildRecord("M", {"coeff": P.Map(P.Map(P.Double())), "const": P.Map(P.Double())})}], P.Map(P.Double()))]) errcodeBase = 31000 def __call__(self, state, scope, pos, paramTypes, datum, model): coeffType = [x["type"] for x in paramTypes[1]["fields"] if x["name"] == "coeff"][0] if coeffType == {'items': 'double', 'type': 'array'}: #sig1 coeff = model["coeff"] + [model["const"]] datum = np().array(datum + [1.0]) if len(datum) != len(coeff): raise PFARuntimeException("misaligned coeff", self.errcodeBase + 0, self.name, pos) return float(np().dot(coeff, datum)) elif coeffType == {'items': {'items': 'double', 'type': 'array'}, 'type': 'array'}: #sig2 if len(model["const"]) != len(model["coeff"]): raise PFARuntimeException("misaligned const", self.errcodeBase + 1, self.name, pos) elif any(len(row) != len(datum) for row in model["coeff"]): raise PFARuntimeException("misaligned coeff", self.errcodeBase + 0, self.name, pos) elif len(datum) == 0: return model["const"] elif len(model["const"]) == 0: return [] else: coeff = np().array(model["coeff"]) const = np().array(model["const"]) datum = np().array(datum + [1.0]) coeff = np().vstack((coeff.T, const)) return list(map(float, np().dot(coeff.T, datum))) elif coeffType == {'values': 'double', 'type': 'map'}: #sig3 coeff = model["coeff"] const = model["const"] out = 0.0 for key in set(list(datum.keys()) + list(coeff.keys())): out += datum.get(key, 0.0) * coeff.get(key, 0.0) return float(out + const) else: coeff = model["coeff"] const = model["const"] outMap = {} innerKeys = set(list(datum.keys()) + sum([list(x.keys()) for x in list(coeff.values())], [])) outerKeys = set(list(const.keys()) + list(coeff.keys())) for outerKey in outerKeys: out = 0.0 for innerKey in innerKeys: out += datum.get(innerKey, 0.0) * coeff.get(outerKey, {}).get(innerKey, 0.0) outMap[outerKey] = float(out + const.get(outerKey, 0.0)) return outMap
class Bin(LibFcn): name = prefix + "bin" sig = Sigs([ Sig([{ "x": P.Double() }, { "numbins": P.Int() }, { "low": P.Double() }, { "high": P.Double() }], P.Int()), Sig([{ "x": P.Double() }, { "origin": P.Double() }, { "width": P.Double() }], P.Int()) ]) errcodeBase = 22000 def __call__(self, state, scope, pos, paramTypes, x, *args): if len(args) == 3: numbins, low, high = args if low >= high or math.isnan(low) or math.isnan(high): raise PFARuntimeException("bad histogram range", self.errcodeBase + 0, self.name, pos) if numbins < 1: raise PFARuntimeException("bad histogram scale", self.errcodeBase + 1, self.name, pos) if math.isnan(x) or x < low or x >= high: raise PFARuntimeException("x out of range", self.errcodeBase + 2, self.name, pos) out = int(math.floor(numbins * div((x - low), (high - low)))) if out < 0 or out >= numbins: raise PFARuntimeException("x out of range", self.errcodeBase + 2, self.name, pos) return out else: origin, width = args if math.isnan(origin) or math.isinf(origin): raise PFARuntimeException("bad histogram range", self.errcodeBase + 0, self.name, pos) if width <= 0.0 or math.isnan(width): raise PFARuntimeException("bad histogram scale", self.errcodeBase + 1, self.name, pos) if math.isnan(x) or math.isinf(x): raise PFARuntimeException("x out of range", self.errcodeBase + 2, self.name, pos) else: return int(math.floor(div((x - origin), width)))
class ZipMap(LibFcn): name = prefix + "zipmap" sig = Sigs([ Sig([{ "a": P.Map(P.Wildcard("A")) }, { "b": P.Map(P.Wildcard("B")) }, { "fcn": P.Fcn([P.Wildcard("A"), P.Wildcard("B")], P.Wildcard("Z")) }], P.Map(P.Wildcard("Z"))), Sig([{ "a": P.Map(P.Wildcard("A")) }, { "b": P.Map(P.Wildcard("B")) }, { "c": P.Map(P.Wildcard("C")) }, { "fcn": P.Fcn([P.Wildcard("A"), P.Wildcard("B"), P.Wildcard("C")], P.Wildcard("Z")) }], P.Map(P.Wildcard("Z"))), Sig([{ "a": P.Map(P.Wildcard("A")) }, { "b": P.Map(P.Wildcard("B")) }, { "c": P.Map(P.Wildcard("C")) }, { "d": P.Map(P.Wildcard("D")) }, { "fcn": P.Fcn([ P.Wildcard("A"), P.Wildcard("B"), P.Wildcard("C"), P.Wildcard("D") ], P.Wildcard("Z")) }], P.Map(P.Wildcard("Z"))) ]) errcodeBase = 26370 def __call__(self, state, scope, pos, paramTypes, *args): fcn = args[-1] maps = args[:-1] keys = maps[0].keys() for m in maps: if keys != m.keys(): raise PFARuntimeException("misaligned maps", self.errcodeBase + 0, self.name, pos) out = {} for k in keys: out[k] = callfcn(state, scope, fcn, [x[k] for x in maps]) return out
class Contains(LibFcn): name = prefix + "contains" sig = Sigs([Sig([{"haystack": P.String()}, {"pattern": P.String()}], P.Boolean()), Sig([{"haystack": P.Bytes()}, {"pattern": P.Bytes()}], P.Boolean())]) errcodeBase = 35010 def __call__(self, state, scope, pos, paramTypes, haystack, pattern): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) out = re.search(0) re.free() return out
class Pull(LibFcn): name = prefix + "pull" sig = Sigs([ Sig([{ "observation": P.Double() }, { "prediction": P.Double() }, { "uncertainty": P.Double() }], P.Double()), Sig([{ "observation": P.Array(P.Double()) }, { "prediction": P.Array(P.Double()) }, { "uncertainty": P.Array(P.Double()) }], P.Array(P.Double())), Sig([{ "observation": P.Map(P.Double()) }, { "prediction": P.Map(P.Double()) }, { "uncertainty": P.Map(P.Double()) }], P.Map(P.Double())) ]) errcodeBase = 38020 def __call__(self, state, scope, pos, paramTypes, observation, prediction, uncertainty): if isinstance(observation, dict): if set(observation.keys()) != set(prediction.keys()): raise PFARuntimeException("misaligned prediction", self.errcodeBase + 0, self.name, pos) if set(observation.keys()) != set(uncertainty.keys()): raise PFARuntimeException("misaligned uncertainty", self.errcodeBase + 1, self.name, pos) return dict( (k, div(observation[k] - prediction[k], uncertainty[k])) for k in observation) elif isinstance(observation, (tuple, list)): if len(observation) != len(prediction): raise PFARuntimeException("misaligned prediction", self.errcodeBase + 0, self.name, pos) if len(observation) != len(uncertainty): raise PFARuntimeException("misaligned uncertainty", self.errcodeBase + 1, self.name, pos) return [ float(div(o - p, u)) for o, p, u in zip(observation, prediction, uncertainty) ] else: return div(observation - prediction, uncertainty)
class Residual(LibFcn): name = prefix + "residual" sig = Sigs([ Sig([{ "observation": P.Double() }, { "prediction": P.Double() }], P.Double(), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.residual instead")), Sig([{ "observation": P.Array(P.Double()) }, { "prediction": P.Array(P.Double()) }], P.Array(P.Double()), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.residual instead")), Sig([{ "observation": P.Map(P.Double()) }, { "prediction": P.Map(P.Double()) }], P.Map(P.Double()), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.residual instead")) ]) errcodeBase = 31020 def __call__(self, state, scope, pos, paramTypes, observation, prediction): if isinstance(observation, dict): if len(observation) != len(prediction): raise PFARuntimeException("misaligned prediction", self.errcodeBase + 0, self.name, pos) result = {} for k, o in observation.items(): try: result[k] = o - prediction[k] except KeyError: raise PFARuntimeException("misaligned prediction", self.errcodeBase + 0, self.name, pos) return result elif isinstance(observation, (tuple, list)): try: result = [ float(o - p) for o, p in zip(observation, prediction) ] except: raise PFARuntimeException("misaligned prediction", self.errcodeBase + 0, self.name, pos) return result else: return float(observation - prediction)
class Sub(LibFcn): name = prefix + "sub" sig = Sigs([ Sig([{ "x": P.Array(P.Double()) }, { "y": P.Array(P.Double()) }], P.Array(P.Double())), Sig([{ "x": P.Array(P.Array(P.Double())) }, { "y": P.Array(P.Array(P.Double())) }], P.Array(P.Array(P.Double()))), Sig([{ "x": P.Map(P.Double()) }, { "y": P.Map(P.Double()) }], P.Map(P.Double())), Sig([{ "x": P.Map(P.Map(P.Double())) }, { "y": P.Map(P.Map(P.Double())) }], P.Map(P.Map(P.Double()))) ]) errcodeBase = 24040 def __call__(self, state, scope, pos, paramTypes, x, y): if isinstance(x, (list, tuple)) and all(isinstance(xi, (list, tuple)) for xi in x) and \ isinstance(y, (list, tuple)) and all(isinstance(yi, (list, tuple)) for yi in y): if len(x) != len(y) or any( len(xi) != len(yi) for xi, yi in zip(x, y)): raise PFARuntimeException("misaligned matrices", self.errcodeBase + 0, self.name, pos) return [[xj - yj for xj, yj in zip(xi, yi)] for xi, yi in zip(x, y)] elif isinstance(x, (list, tuple)) and isinstance(y, (list, tuple)): if len(x) != len(y): raise PFARuntimeException("misaligned matrices", self.errcodeBase + 0, self.name, pos) return [xi - yi for xi, yi in zip(x, y)] elif isinstance(x, dict) and all(isinstance(x[i], dict) for i in x.keys()) and \ isinstance(y, dict) and all(isinstance(y[i], dict) for i in y.keys()): rows = rowKeys(x).union(rowKeys(y)) cols = colKeys(x).union(colKeys(y)) return dict( (i, dict((j, x.get(i, {}).get(j, 0.0) - y.get(i, {}).get(j, 0.0)) for j in cols)) for i in rows) else: rows = rowKeys(x).union(rowKeys(y)) return dict((i, x.get(i, 0.0) - y.get(i, 0.0)) for i in rows)
class MapApply(LibFcn): name = prefix + "map" sig = Sigs([Sig([{"x": P.Array(P.Array(P.Double()))}, {"fcn": P.Fcn([P.Double()], P.Double())}], P.Array(P.Array(P.Double()))), Sig([{"x": P.Map(P.Map(P.Double()))}, {"fcn": P.Fcn([P.Double()], P.Double())}], P.Map(P.Map(P.Double())))]) errcodeBase = 24000 def __call__(self, state, scope, pos, paramTypes, x, fcn): if isinstance(x, (list, tuple)) and all(isinstance(xi, (list, tuple)) for xi in x): return [[callfcn(state, scope, fcn, [xj]) for xj in xi] for xi in x] elif isinstance(x, dict) and all(isinstance(x[i], dict) for i in x.keys()): return dict((i, dict((j, callfcn(state, scope, fcn, [xj])) for j, xj in xi.items())) for i, xi in x.items())
class LinearMissing(LibFcn): name = prefix + "linearMissing" sig = Sigs([Sig([{"x": P.Double()}, {"table": P.Array(P.WildRecord("R", {"x": P.Double(), "to": P.Double()}))}], P.Union([P.Null(), P.Double()])), Sig([{"x": P.Double()}, {"table": P.Array(P.WildRecord("R", {"x": P.Double(), "to": P.Array(P.Double())}))}], P.Union([P.Null(), P.Array(P.Double())]))]) errcodeBase = 22040 def __call__(self, state, scope, pos, paramTypes, datum, table): one, two, between = Linear.closest(datum, table, self.errcodeBase + 0, self.name, pos) if not between and one["x"] != datum: return None elif isinstance([x for x in paramTypes[-1] if x != "null"][0], dict) and [x for x in paramTypes[-1] if x != "null"][0].get("type") == "array": return Linear.interpolateMulti(datum, one, two, self.errcodeBase + 1, self.name, pos) else: return Linear.interpolateSingle(datum, one, two)
class LinearFlat(LibFcn): name = prefix + "linearFlat" sig = Sigs([Sig([{"x": P.Double()}, {"table": P.Array(P.WildRecord("R", {"x": P.Double(), "to": P.Double()}))}], P.Double()), Sig([{"x": P.Double()}, {"table": P.Array(P.WildRecord("R", {"x": P.Double(), "to": P.Array(P.Double())}))}], P.Array(P.Double()))]) errcodeBase = 22030 def __call__(self, state, scope, pos, paramTypes, datum, table): one, two, between = Linear.closest(datum, table, self.errcodeBase + 0, self.name, pos) if not between: return one["to"] elif isinstance(paramTypes[-1], dict) and paramTypes[-1].get("type") == "array": return Linear.interpolateMulti(datum, one, two, self.errcodeBase + 1, self.name, pos) else: return Linear.interpolateSingle(datum, one, two)
class UpdateChi2(LibFcn): name = prefix + "updateChi2" sig = Sigs([Sig([{"pull": P.Double()}, {"state_": P.WildRecord("A", {"chi2": P.Double(), "DOF": P.Int()})}], P.Wildcard("A"), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.updateChi2 instead")), Sig([{"pull": P.Array(P.Double())}, {"state_": P.WildRecord("A", {"chi2": P.Double(), "DOF": P.Int()})}], P.Wildcard("A"), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.updateChi2 instead")), Sig([{"pull": P.Map(P.Double())}, {"state_": P.WildRecord("A", {"chi2": P.Double(), "DOF": P.Int()})}], P.Wildcard("A"), Lifespan(None, PFAVersion(0, 7, 2), PFAVersion(0, 9, 0), "use test.updateChi2 instead"))]) errcodeBase = 31050 def __call__(self, state, scope, pos, paramTypes, pull, state_): if isinstance(pull, float): return update(pull*pull, state_) elif isinstance(pull, (tuple, list)): return update(sum([y**2 for y in pull]), state_) else: return update(sum([y**2 for y in list(pull.values())]), state_)
class StartsWith(LibFcn): name = prefix + "startswith" sig = Sigs([Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Array(P.Wildcard("A"))}], P.Boolean()), Sig([{"haystack": P.Array(P.Wildcard("A"))}, {"needle": P.Wildcard("A")}], P.Boolean())]) errcodeBase = 15110 def __call__(self, state, scope, pos, paramTypes, haystack, needle): if isinstance(needle, (list, tuple)): return needle == haystack[:len(needle)] else: if len(haystack) == 0: return False else: return needle == haystack[0]
class Det(LibFcn): name = prefix + "det" sig = Sigs([ Sig([{ "x": P.Array(P.Array(P.Double())) }], P.Double()), Sig([{ "x": P.Map(P.Map(P.Double())) }], P.Double()) ]) errcodeBase = 24090 def __call__(self, state, scope, pos, paramTypes, x): if isinstance(x, (list, tuple)) and all( isinstance(xi, (list, tuple)) for xi in x): rows = len(x) if rows < 1: raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) cols = len(x[0]) if cols < 1: raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) if raggedArray(x): raise PFARuntimeException("ragged columns", self.errcodeBase + 1, self.name, pos) if rows != cols: raise PFARuntimeException("non-square matrix", self.errcodeBase + 2, self.name, pos) if any( any(math.isnan(z) or math.isinf(z) for z in row) for row in x): return float("nan") else: return float(np().linalg.det(arraysToMatrix(x))) elif isinstance(x, dict) and all( isinstance(x[i], dict) for i in list(x.keys())): keys = list(rowKeys(x).union(colKeys(x))) if len(keys) < 1 or all(len(row) == 0 for row in list(x.values())): raise PFARuntimeException("too few rows/cols", self.errcodeBase + 0, self.name, pos) if any( any( math.isnan(z) or math.isinf(z) for z in list(row.values())) for row in list(x.values())): return float("nan") else: return float(np().linalg.det(mapsToMatrix(x, keys, keys)))
class IsNan(LibFcn): name = prefix + "isnan" sig = Sigs([ Sig([{ "x": P.Float() }], P.Boolean()), Sig([{ "x": P.Double() }], P.Boolean()) ]) errcodeBase = 21020 def __call__(self, state, scope, pos, paramTypes, x): return math.isnan(x)
class ZipMapWithIndex(LibFcn): name = prefix + "zipmapWithIndex" sig = Sigs([Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("B"))}, {"fcn": P.Fcn([P.Int(), P.Wildcard("A"), P.Wildcard("B")], P.Wildcard("Z"))}], P.Array(P.Wildcard("Z"))), Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("B"))}, {"c": P.Array(P.Wildcard("C"))}, {"fcn": P.Fcn([P.Int(), P.Wildcard("A"), P.Wildcard("B"), P.Wildcard("C")], P.Wildcard("Z"))}], P.Array(P.Wildcard("Z"))), Sig([{"a": P.Array(P.Wildcard("A"))}, {"b": P.Array(P.Wildcard("B"))}, {"c": P.Array(P.Wildcard("C"))}, {"d": P.Array(P.Wildcard("D"))}, {"fcn": P.Fcn([P.Int(), P.Wildcard("A"), P.Wildcard("B"), P.Wildcard("C"), P.Wildcard("D")], P.Wildcard("Z"))}], P.Array(P.Wildcard("Z")))]) errcodeBase = 15660 def __call__(self, state, scope, pos, paramTypes, *args): if len(set(map(len, args[:-1]))) != 1: raise PFARuntimeException("misaligned arrays", self.errcodeBase + 0, self.name, pos) fcn = args[-1] out = [] for i, abcd in enumerate(zip(*args[:-1])): out.append(callfcn(state, scope, fcn, [i] + list(abcd))) return out
class NearestK(LibFcn): name = prefix + "nearestK" sig = Sigs([ Sig([{ "k": P.Int() }, { "datum": P.Array(P.Double()) }, { "codebook": P.Array(P.Array(P.Double())) }], P.Array(P.Array(P.Double()))), Sig([{ "k": P.Int() }, { "datum": P.Wildcard("A") }, { "codebook": P.Array(P.Wildcard("B")) }, { "metric": P.Fcn([P.Wildcard("A"), P.Wildcard("B")], P.Double()) }], P.Array(P.Wildcard("B"))) ]) errcodeBase = 30010 def __call__(self, state, scope, pos, paramTypes, k, datum, codebook, *args): if k < 0: raise PFARuntimeException("k must be nonnegative", self.errcodeBase + 0, self.name, pos) if len(args) == 1: metric, = args distances = [ callfcn(state, scope, metric, [datum, x]) for x in codebook ] else: if len(codebook) == 0: return [] else: dimensions = len(datum) for x in codebook: if len(x) != dimensions: raise PFARuntimeException( "inconsistent dimensionality", self.errcodeBase + 1, self.name, pos) distances = [ sum((di - xi)**2 for di, xi in zip(datum, x)) for x in codebook ] indexes = argLowestN(distances, k, lambda a, b: a < b) return [codebook[i] for i in indexes]
class RandomBytes(LibFcn): name = prefix + "bytes" sig = Sigs([ Sig([{ "size": P.Int() }], P.Bytes()), Sig([{ "size": P.Int() }, { "population": P.Bytes() }], P.Bytes()), Sig([{ "size": P.Int() }, { "low": P.Int() }, { "high": P.Int() }], P.Bytes()) ]) errcodeBase = 34090 def __call__(self, state, scope, pos, paramTypes, size, *args): if size <= 0: raise PFARuntimeException("size must be positive", self.errcodeBase + 0, self.name, pos) if len(args) == 0: return "".join( chr(state.rand.randint(0, 255)) for x in range(size)) elif len(args) == 1: if len(args[0]) == 0: raise PFARuntimeException("population must be non-empty", self.errcodeBase + 3, self.name, pos) if isinstance(args[0], str): population = list(args[0]) elif isinstance(args[0], bytes): population = list(bytesToString(args[0])) return "".join(population[state.rand.randint(0, len(args[0]) - 1)] for x in range(size)) else: low, high = args if high <= low: raise PFARuntimeException("high must be greater than low", self.errcodeBase + 1, self.name, pos) if low < 0 or low > 255 or high < 0 or high > 256: raise PFARuntimeException("invalid byte", self.errcodeBase + 2, self.name, pos) return "".join( chr(state.rand.randint(low, high - 1)) for x in range(size))
class ErrorOnNonNum(LibFcn): name = prefix + "errorOnNonNum" sig = Sigs([Sig([{"x": P.Float()}], P.Float()), Sig([{"x": P.Double()}], P.Double())]) errcodeBase = 21050 def __call__(self, state, scope, pos, paramTypes, x): if math.isnan(x): raise PFARuntimeException("encountered nan", self.errcodeBase + 0, self.name, pos) elif math.isinf(x): if x > 0.0: raise PFARuntimeException("encountered +inf", self.errcodeBase + 1, self.name, pos) else: raise PFARuntimeException("encountered -inf", self.errcodeBase + 2, self.name, pos) else: return x
class Index(LibFcn): name = prefix + "index" sig = Sigs([Sig([{"haystack": P.String()}, {"pattern": P.String()}], P.Array(P.Int())), Sig([{"haystack": P.Bytes()}, {"pattern": P.Bytes()}], P.Array(P.Int()))]) errcodeBase = 35000 def __call__(self, state, scope, pos, paramTypes, haystack, pattern): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) if re.search(0): region = re.getRegion() out = [region.beg[0], region.end[0]] else: out = [] re.free() return out
class LinearVariance(LibFcn): name = prefix + "linearVariance" sig = Sigs([Sig([{"datum": P.Array(P.Double())}, {"model": P.WildRecord("M", {"covar": P.Array(P.Array(P.Double()))})}], P.Double()), Sig([{"datum": P.Array(P.Double())}, {"model": P.WildRecord("M", {"covar": P.Array(P.Array(P.Array(P.Double())))})}], P.Array(P.Double())), Sig([{"datum": P.Map(P.Double())}, {"model": P.WildRecord("M", {"covar": P.Map(P.Map(P.Double()))})}], P.Double()), Sig([{"datum": P.Map(P.Double())}, {"model": P.WildRecord("M", {"covar": P.Map(P.Map(P.Map(P.Double())))})}], P.Map(P.Double()))]) errcodeBase = 31010 def __call__(self, state, scope, pos, paramTypes, datum, model): covarType = [x["type"] for x in paramTypes[1]["fields"] if x["name"] == "covar"][0] if covarType == {"type": "array", "items": {"type": "array", "items": "double"}}: # sig1 datum = datum + [1.0] covar = model["covar"] if len(datum) != len(covar) or any(len(datum) != len(x) for x in covar): raise PFARuntimeException("misaligned covariance", self.errcodeBase + 0, self.name, pos) x = np().matrix([datum]) C = np().matrix(covar) return float(x.dot(C.dot(x.T))[0][0]) elif covarType == {"type": "array", "items": {"type": "array", "items": {"type": "array", "items": "double"}}}: # sig2 datum = datum + [1.0] x = np().matrix([datum]) out = [] for covar in model["covar"]: if len(datum) != len(covar) or any(len(datum) != len(x) for x in covar): raise PFARuntimeException("misaligned covariance", self.errcodeBase + 0, self.name, pos) C = np().matrix(covar) out.append(float(x.dot(C.dot(x.T))[0][0])) return out elif covarType == {"type": "map", "values": {"type": "map", "values": "double"}}: # sig3 datum = dict(list(datum.items()) + [("", 1.0)]) covar = model["covar"] keys = list(set(list(datum.keys()) + sum([list(x.keys()) for x in list(covar.values())], []))) x = np().matrix([[datum.get(k, 0.0) for k in keys]]) C = np().matrix([[covar.get(i, {}).get(j, 0.0) for j in keys] for i in keys]) return float(x.dot(C.dot(x.T))[0][0]) else: datum = dict(list(datum.items()) + [("", 1.0)]) covar = model["covar"] keys = list(set(list(datum.keys()) + sum(flatten([[list(x.keys()) for x in list(row.values())] for row in list(covar.values())]), []))) x = np().matrix([[datum.get(k, 0.0) for k in keys]]) out = {} for depkey, row in list(covar.items()): C = np().matrix([[row.get(i, {}).get(j, 0.0) for j in keys] for i in keys]) out[depkey] = float(x.dot(C.dot(x.T))[0][0]) return out
class ZValue(LibFcn): name = prefix + "zValue" sig = Sigs([ Sig([{ "x": P.Double() }, { "meanVariance": P.WildRecord("A", { "mean": P.Double(), "variance": P.Double() }) }], P.Double()), Sig([{ "x": P.Double() }, { "meanVariance": P.WildRecord("A", { "count": P.Double(), "mean": P.Double(), "variance": P.Double() }) }, { "unbiased": P.Boolean() }], P.Double()) ]) errcodeBase = 37010 def __call__(self, state, scope, pos, paramTypes, x, meanVariance, *args): mean = meanVariance["mean"] variance = meanVariance["variance"] if variance >= 0.0: sigma = math.sqrt(variance) else: sigma = float("nan") if len(args) == 0: return div(x - mean, sigma) else: unbiased, = args count = meanVariance["count"] if count / (count - 1.0) >= 0.0: correction = math.sqrt((count) / (count - 1.0)) else: correction = float("nan") if unbiased: return div(x - mean, sigma) * correction else: return div(x - mean, sigma)
class BitwiseNot(LibFcn): name = "~" sig = Sigs( [Sig([{ "x": P.Int() }], P.Int()), Sig([{ "x": P.Long() }], P.Long())]) errcodeBase = 18280 def genpy(self, paramTypes, args, pos): return "(~{0})".format(*args) def __call__(self, state, scope, pos, paramTypes, x): return ~x
class Scale(LibFcn): name = prefix + "scale" sig = Sigs([Sig([{"x": P.Array(P.Double())}, {"alpha": P.Double()}], P.Array(P.Double())), Sig([{"x": P.Array(P.Array(P.Double()))}, {"alpha": P.Double()}], P.Array(P.Array(P.Double()))), Sig([{"x": P.Map(P.Double())}, {"alpha": P.Double()}], P.Map(P.Double())), Sig([{"x": P.Map(P.Map(P.Double()))}, {"alpha": P.Double()}], P.Map(P.Map(P.Double())))]) errcodeBase = 24010 def __call__(self, state, scope, pos, paramTypes, x, alpha): if isinstance(x, (list, tuple)) and all(isinstance(xi, (list, tuple)) for xi in x): return [[xj * alpha for xj in xi] for xi in x] elif isinstance(x, (list, tuple)): return [xi * alpha for xi in x] elif isinstance(x, dict) and all(isinstance(x[i], dict) for i in x): return dict((i, dict((j, xj * alpha) for j, xj in xi.items())) for i, xi in x.items()) else: return dict((i, xi * alpha) for i, xi in x.items())
class ReplaceFirst(LibFcn): name = prefix + "replacefirst" sig = Sigs([Sig([{"haystack": P.String()}, {"pattern": P.String()}, {"replacement": P.String()}], P.String()), Sig([{"haystack": P.Bytes()}, {"pattern": P.Bytes()}, {"replacement": P.Bytes()}], P.Bytes())]) errcodeBase = 35110 def __call__(self, state, scope, pos, paramTypes, haystack, pattern, replacement): haystack, pattern, to = convert(haystack, pattern, paramTypes[0]) re = Regexer(haystack, pattern, self.errcodeBase + 0, self.name, pos) found = re.search(0) region = re.getRegion() if found: out = to(haystack[:region.beg[0]]) + replacement + to(haystack[region.end[0]:]) else: out = to(haystack) re.free() return out
class ClosestN(LibFcn): name = prefix + "closestN" sig = Sigs([ Sig([{"n": P.Int()}, {"datum": P.Array(P.Double())}, {"clusters": P.Array(P.WildRecord("C", {"center": P.Array(P.Double())}))}], P.Array(P.Wildcard("C"))), Sig([{"n": P.Int()}, {"datum": P.Wildcard("A")}, {"clusters": P.Array(P.WildRecord("C", {"center": P.Wildcard("B")}))}, {"metric": P.Fcn([P.Wildcard("A"), P.Wildcard("B")], P.Double())}], P.Array(P.Wildcard("C")))]) errcodeBase = 29010 def __call__(self, state, scope, pos, paramTypes, n, datum, clusters, *args): if n < 0: raise PFARuntimeException("n must be nonnegative", self.errcodeBase + 0, self.name, pos) if len(args) == 1: metric, = args distances = [callfcn(state, scope, metric, [datum, x["center"]]) for x in clusters] else: distances = [sum((di - xi)**2 for di, xi in zip(datum, x["center"])) for x in clusters] indexes = argLowestN(distances, n, lambda a, b: a < b) return [clusters[i] for i in indexes]
class Hex(LibFcn): name = prefix + "hex" sig = Sigs([ Sig([{ "x": P.Long() }], P.String()), Sig([{ "x": P.Long() }, { "width": P.Int() }, { "zeroPad": P.Boolean() }], P.String()) ]) errcodeBase = 39110 def __call__(self, state, scope, pos, paramTypes, *args): if len(args) == 1: x, = args if x < 0: raise PFARuntimeException("negative number", self.errcodeBase + 1, self.name, pos) else: return "{0:x}".format(x) else: x, width, zeroPad = args if x < 0: raise PFARuntimeException("negative number", self.errcodeBase + 1, self.name, pos) if not zeroPad: if width < 0: formatStr = "{0:<" + str(-width) + "x}" elif width == 0: formatStr = "" else: formatStr = "{0:" + str(width) + "x}" else: if width < 0: raise PFARuntimeException( "negative width cannot be used with zero-padding", self.errcodeBase + 0, self.name, pos) elif width == 0: formatStr = "" else: formatStr = "{0:0" + str(width) + "x}" return formatStr.format(x)