def testCauchit(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.cauchit: input} """) self.assertAlmostEqual(engine.action(2.2), 0.8642002512199081, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.cauchit: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 0.75 , places = 5) self.assertAlmostEqual(out[1], 0.8524163823495667, places = 5) self.assertAlmostEqual(out[2], 0.8975836176504333, places = 5) self.assertAlmostEqual(out[3], 0.9220208696226307, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.cauchit: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.75 , places = 5) self.assertAlmostEqual(out["two"], 0.8524163823495667, places = 5) self.assertAlmostEqual(out["three"], 0.8975836176504333, places = 5) self.assertAlmostEqual(out["four"], 0.9220208696226307, places = 5)
def testMakeTimestamp(self): engine, = PFAEngine.fromYaml(""" input: int output: double action: - {time.makeTimestamp: [input, 3, 11, 14, 50, 6, 245, {string: ""}]} """) self.assertEqual(engine.action(2015), 1426085406.245) engine2, = PFAEngine.fromYaml(""" input: int output: double action: - {time.makeTimestamp: [input, 1, 1, 0, 0, 0, 0, {string: ""}]} """) self.assertEqual(engine2.action(1970), 0) engine3, = PFAEngine.fromYaml(""" input: string output: double action: - {time.makeTimestamp: [1970, 1, 1, 0, 0, 0, 0, input]} """) self.assertEqual(engine3.action("Europe/Paris"), -3600) self.assertEqual(engine3.action("Etc/UTC"), 0) self.assertEqual(engine3.action("Atlantic/Azores"), 3600)
def testNChooseKFcn(self): engine, = PFAEngine.fromYaml(''' input: int output: int action: - m.special.nChooseK: [input, 2] ''') self.assertEqual(engine.action(20), 190) self.assertEqual(engine.action(10), 45) self.assertEqual(engine.action(3), 3) ### raise the right exceptions ### engine, = PFAEngine.fromYaml(''' input: int output: int action: - m.special.nChooseK: [input, 4] ''') self.assertRaises(PFARuntimeException, lambda: engine.action(1)) self.assertRaises(PFARuntimeException, lambda: engine.action(0)) engine, = PFAEngine.fromYaml(''' input: int output: int action: - m.special.nChooseK: [input, -4] #[input, lambda] ''') self.assertRaises(PFARuntimeException, lambda: engine.action(-2))
def testDoAbs(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {m.abs: input} ''') self.assertAlmostEqual(engine.action(-3.14), 3.14, places=2) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {m.abs: input} ''') self.assertEqual(engine.action(2147483647), 2147483647) self.assertEqual(engine.action(-2147483647), 2147483647) self.assertRaises(PFARuntimeException, lambda: engine.action(-2147483648)) engine, = PFAEngine.fromYaml(''' input: long output: long action: - {m.abs: input} ''') self.assertEqual(engine.action(9223372036854775807), 9223372036854775807) self.assertEqual(engine.action(-9223372036854775807), 9223372036854775807) self.assertRaises(PFARuntimeException, lambda: engine.action(-9223372036854775808))
def testRegMahalanobis(self): engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: double action: stat.test.mahalanobis: - input - type: {type: array, items: double} value: [2.5, 2.5, 2.5] - type: {type: array, items: {type: array, items: double}} value: [[2.0, 0.0, 0.0], [0.0, 4.0, 0.0], [0.0, 0.0, 1.0]] """) self.assertAlmostEqual(engine.action([1, 2, 3]), 1.19895788083, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: double action: stat.test.mahalanobis: - input - type: {type: map, values: double} value: {one: 2.5, two: 2.5, three: 2.5} - type: {type: map, values: {type: map, values: double}} value: {one: {one: 2.0, two: 0.0, three: 0.0}, two: {one: 0.0, two: 4.0, three: 0.0}, three: {one: 0.0, two: 0.0, three: 1.0}} """) self.assertAlmostEqual(engine.action({"one": 1, "two": 2, "three": 3}), 1.19895788083, places = 5)
def testSoftPlus(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.softplus: input} """) self.assertAlmostEqual(engine.action(2.2), 2.305083319768696, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.softplus: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 1.31326168752 , places = 5) self.assertAlmostEqual(out[1], 2.12692801104 , places = 5) self.assertAlmostEqual(out[2], 3.04858735157 , places = 5) self.assertAlmostEqual(out[3], 4.01814992792 , places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.softplus: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 1.31326168752, places = 5) self.assertAlmostEqual(out["two"], 2.12692801104, places = 5) self.assertAlmostEqual(out["three"], 3.04858735157, places = 5) self.assertAlmostEqual(out["four"], 4.01814992792, places = 5)
def testDoMin(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {min: [input, 3.2]} ''') self.assertEqual(engine.action(2.2), 2.2) self.assertEqual(engine.action(4.2), 3.2) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {min: [input, 3]} ''') self.assertEqual(engine.action(2), 2) self.assertEqual(engine.action(4), 3) engine, = PFAEngine.fromYaml(''' input: string output: string action: - {min: [input, [HAL]]} ''') self.assertEqual(engine.action("GZK"), "GZK") self.assertEqual(engine.action("IBM"), "HAL")
def testCloglog(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.cloglog: input} """) self.assertAlmostEqual(engine.action(2.2), 0.9998796388196516, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.cloglog: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 0.9340119641546875, places = 5) self.assertAlmostEqual(out[1], 0.9993820210106689, places = 5) self.assertAlmostEqual(out[2], 0.9999999981078213, places = 5) self.assertAlmostEqual(out[3], 1.0 , places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.cloglog: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.9340119641546875, places = 5) self.assertAlmostEqual(out["two"], 0.9993820210106689, places = 5) self.assertAlmostEqual(out["three"], 0.9999999981078213, places = 5) self.assertAlmostEqual(out["four"], 1.0 , places = 5)
def testBitwiseOperators(self): engine, = PFAEngine.fromYaml(''' input: "null" output: int action: - {"&": [85, 15]} ''') self.assertEqual(engine.action(None), 5) engine, = PFAEngine.fromYaml(''' input: "null" output: int action: - {"|": [85, 15]} ''') self.assertEqual(engine.action(None), 95) engine, = PFAEngine.fromYaml(''' input: "null" output: int action: - {"^": [85, 15]} ''') self.assertEqual(engine.action(None), 90) engine, = PFAEngine.fromYaml(''' input: "null" output: int action: - {"~": [85]} ''') self.assertEqual(engine.action(None), -86)
def testDoCmp(self): engine, = PFAEngine.fromYaml(''' input: int output: int action: - {cmp: [input, 5]} ''') self.assertEqual(engine.action(3), -1) self.assertEqual(engine.action(5), 0) self.assertEqual(engine.action(7), 1) engine, = PFAEngine.fromYaml(''' input: double output: int action: - {cmp: [input, 5.3]} ''') self.assertEqual(engine.action(5.2), -1) self.assertEqual(engine.action(5.3), 0) self.assertEqual(engine.action(5.4), 1) engine, = PFAEngine.fromYaml(''' input: string output: int action: - {cmp: [input, [HAL]]} ''') self.assertEqual(engine.action("GZK"), -1) self.assertEqual(engine.action("HAL"), 0) self.assertEqual(engine.action("IBM"), 1)
def testRejectComparisonsBetweenDifferentTypesEvenIfTheyHaveTheSameStructure(self): engine, = PFAEngine.fromYaml(''' input: {type: enum, name: Category1, symbols: [z, y, x, w]} output: boolean action: - {"==": [input, {type: Category1, value: x}]} ''') self.assertFalse(engine.action("z")) self.assertFalse(engine.action("y")) self.assertTrue(engine.action("x")) self.assertFalse(engine.action("w")) self.assertRaises(PFASemanticException, lambda: PFAEngine.fromYaml(''' input: {type: enum, name: Category1, symbols: [z, y, x, w]} output: boolean action: - {"==": [input, {type: {type: enum, name: Category2, symbols: [w, x, y, z]}, value: x}]} ''')) self.assertRaises(PFASemanticException, lambda: PFAEngine.fromYaml(''' input: {type: enum, name: Category1, symbols: [z, y, x, w]} output: boolean action: - {"==": [input, {type: {type: enum, name: Category2, symbols: [w, x, y, whatever]}, value: x}]} '''))
def testBin(self): engine1, = PFAEngine.fromYaml(''' input: double output: int action: interp.bin: [input, 5, 100, 110] ''') self.assertEqual(engine1.action(100.0), 0) self.assertEqual(engine1.action(101.0), 0) self.assertEqual(engine1.action(101.999), 0) self.assertEqual(engine1.action(102.0), 1) self.assertEqual(engine1.action(109.999), 4) engine2, = PFAEngine.fromYaml(''' input: double output: int action: interp.bin: [input, 100, 2] ''') self.assertEqual(engine2.action(99.0), -1) self.assertEqual(engine2.action(99.999), -1) self.assertEqual(engine2.action(100.0), 0) self.assertEqual(engine2.action(101.0), 0) self.assertEqual(engine2.action(101.999), 0) self.assertEqual(engine2.action(102.0), 1) self.assertEqual(engine2.action(109.999), 4) self.assertEqual(engine2.action(110.0), 5) self.assertEqual(engine2.action(0.0), -50)
def testDoExponentiation(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {"**": [input, 30]} ''') self.assertAlmostEqual(engine.action(2.5), 867361737988.4036, places=2) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {"**": [input, 30]} ''') self.assertEqual(engine.action(2), 1073741824) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {"**": [input, 30]} ''') self.assertRaises(PFARuntimeException, lambda: engine.action(3)) engine, = PFAEngine.fromYaml(''' input: long output: long action: - {"**": [input, 30]} ''') self.assertEqual(engine.action(3), 205891132094649)
def testgroupsAll(self): engine, = PFAEngine.fromYaml(""" input: string output: {type: array, items: {type: array, items: {type: array, items: int}}} action: - {re.groupsall: [input, [()(a)bc(def)ghijk]]} """) self.assertEqual(engine.action("abcdefghijkMMMMMabcdefghijkMMMM"), [[[0,11], [0,0], [0,1], [3,6]], [[16, 27],[16,16],[16,17], [19,22]]]) ## check non-ascii input # engine, = PFAEngine.fromYaml(""" #input: string #output: {type: array, items: {type: array, items: {type: array, items: int}}} #action: # - {re.groupsall: [input, [(机)机]]} #""") # self.assertEqual(engine.action("abc机机abca机机bc"), [[[3,5], [3,4]], [[9,11], [9,10]]]) # check byte input engine, = PFAEngine.fromYaml(""" input: bytes output: {type: array, items: {type: array, items: {type: array, items: int}}} action: - re.groupsall: [input, {bytes.encodeUtf8: {string: "(机)机"}}] """) self.assertEqual(engine.action("abc机机abca机机bc"), [[[3,9], [3,6]], [[13,19], [13,16]]])
def testfindFirst(self): engine, = PFAEngine.fromYaml(""" input: string output: [string, "null"] action: - {re.findfirst: [input, [ab]]} """) self.assertEqual(engine.action("88ccc555"), None) self.assertEqual(engine.action("abcabcabc"), "ab") # check non-ascii input # engine, = PFAEngine.fromYaml(""" #input: string #output: [string, "null"] #action: # - {re.findfirst: [input, [机机+]]} #""") # self.assertEqual(engine.action("abc机机机abca机机bc asdkj 机机机sd"), "机机机") # self.assertEqual(engine.action("abdefg"), None) # check byte input engine, = PFAEngine.fromYaml(""" input: bytes output: [bytes, "null"] action: - re.findfirst: [input, {bytes.encodeUtf8: {string: "对讲机(讲|机)*"}}] """) self.assertEqual(engine.action("abcde对讲机讲fgg对讲机讲h"), "对讲机讲") self.assertEqual(engine.action("abcdefghijk"), None)
def testBytes(self): engine1, = PFAEngine.fromYaml(''' input: "null" output: bytes randseed: 12345 action: {rand.bytes: [10]} ''') self.assertEqual(engine1.action(None), "j\x02\xd3L^1\x90)\x1fn") self.assertEqual(engine1.action(None), "\x8f,\x8dZ\xf5\x17\xfai\x81%") self.assertEqual(engine1.action(None), "\xb80W\x06V\xf7\xfa\xbe\x00\xf0") engine2, = PFAEngine.fromYaml(''' input: "null" output: bytes randseed: 12345 action: {rand.bytes: [10, {base64: "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5"}]} ''') self.assertEqual(engine2.action(None), "oa3kngufep") self.assertEqual(engine2.action(None), "ugtm8d9osf") self.assertEqual(engine2.action(None), "zgmam890a7") engine3, = PFAEngine.fromYaml(''' input: "null" output: bytes randseed: 12345 action: {rand.bytes: [10, 33, 127]} ''') self.assertEqual(engine3.action(None), "H!n=C3V0,I") self.assertEqual(engine3.action(None), "U1UB{)|GP.") self.assertEqual(engine3.action(None), "d2A#@{}f!y")
def testUUID(self): engine1, = PFAEngine.fromYaml(''' input: "null" output: string randseed: 12345 action: {rand.uuid4: []} ''') self.assertEqual(engine1.action(None), "6aa79987-bb91-4029-8d1f-cd8778e7d340bbcd") self.assertEqual(engine1.action(None), "4c73a942-daea-45e5-8ee8-452ec40a3193ca54") self.assertEqual(engine1.action(None), "90e5e945-6fac-4296-85f8-dfc9e3b11fcff454") engine2, = PFAEngine.fromYaml(''' input: "null" output: string action: {s.substr: [{rand.uuid4: []}, 14, 15]} ''') for i in xrange(1000): self.assertEqual(engine2.action(None), "4") engine3, = PFAEngine.fromYaml(''' input: "null" output: string action: {s.substr: [{rand.uuid4: []}, 19, 20]} ''') for i in xrange(1000): self.assertEqual(engine3.action(None), "8")
def testreplaceLast(self): engine, = PFAEngine.fromYaml(""" input: string output: string action: - {re.replacelast: [input, ["ab(c|d)*"], ["person"]]} """) self.assertEqual(engine.action("abcccdcPPPPabcccdc"), "abcccdcPPPPperson") self.assertEqual(engine.action("abcccdcPPPPabcccdcPPPP"), "abcccdcPPPPpersonPPPP") # check non-ascii input # engine, = PFAEngine.fromYaml(""" #input: string #output: string #action: # - {re.replacelast: [input, [对讲机+], ["walkie talkie"]]} #""") # self.assertEqual(engine.action("This 对讲机 works better than that 对讲机."), "This 对讲机 works better than that walkie talkie.") # check byte input engine, = PFAEngine.fromYaml(""" input: bytes output: bytes action: - {re.replacelast: [input, {bytes.encodeUtf8: {string: "对讲机+"}}, {bytes.encodeUtf8: {string: "walkie talkie"}}]} """) self.assertEqual(engine.action("This 对讲机 works better than that 对讲机."), "This 对讲机 works better than that walkie talkie.")
def bad(): PFAEngine.fromYaml(''' input: [int, string, "null"] output: int action: - {impute.defaultOnNull: [input, 12]} ''')
def testString(self): engine1, = PFAEngine.fromYaml(''' input: "null" output: string randseed: 12345 action: {rand.string: [10]} ''') self.assertEqual(engine1.action(None), u"姾ȳ눿䂂侔⧕穂⋭嶄") self.assertEqual(engine1.action(None), u"祩▩睿䲩컲Ꮉ퍣夅泚 ") self.assertEqual(engine1.action(None), u"魍⤉䧇ԕ䥖탺퍬ꃒÀ쬘") engine2, = PFAEngine.fromYaml(''' input: "null" output: string randseed: 12345 action: {rand.string: [10, {string: "abcdefghijklmnopqrstuvwxyz0123456789"}]} ''') self.assertEqual(engine2.action(None), "oa3kngufep") self.assertEqual(engine2.action(None), "ugtm8d9osf") self.assertEqual(engine2.action(None), "zgmam890a7") engine3, = PFAEngine.fromYaml(''' input: "null" output: string randseed: 12345 action: {rand.string: [10, 33, 127]} ''') self.assertEqual(engine3.action(None), "H!n=C3V0,I") self.assertEqual(engine3.action(None), "U1UB{)|GP.") self.assertEqual(engine3.action(None), "d2A#@{}f!y")
def testScaleMaps(self): engine, = PFAEngine.fromYaml(''' input: "null" output: {type: map, values: double} action: la.scale: - type: {type: map, values: double} value: {one: 1, two: 2, three: 3, four: 4, five: 5} - 10 ''') self.assertAlmostEqual(self.chi2Vector(engine.action(None), {"one": 10.0, "two": 20.0, "three": 30.0, "four": 40.0, "five": 50.0}), 0.0, places=2) engine, = PFAEngine.fromYaml(''' input: "null" output: {type: map, values: {type: map, values: double}} action: la.scale: - type: {type: map, values: {type: map, values: double}} value: {uno: {one: 1, two: 2, three: 3}, dos: {four: 4, five: 5, six: 6}, tres: {seven: 7, eight: 8, nine: 9}} - 10 ''') self.assertAlmostEqual(self.chi2(engine.action(None), {"uno": {"one": 10.0, "two": 20.0, "three": 30.0}, "dos": {"four": 40.0, "five": 50.0, "six": 60.0}, "tres": {"seven": 70.0, "eight": 80.0, "nine": 90.0}}), 0.0, places=2)
def testLogit(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.logit: input} """) self.assertAlmostEqual(engine.action(2.2), 0.9002495108803148, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.logit: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 0.7310585786300049, places = 5) self.assertAlmostEqual(out[1], 0.8807970779778823, places = 5) self.assertAlmostEqual(out[2], 0.9525741268224334, places = 5) self.assertAlmostEqual(out[3], 0.9820137900379085, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.logit: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.7310585786300049, places = 5) self.assertAlmostEqual(out["two"], 0.8807970779778823, places = 5) self.assertAlmostEqual(out["three"],0.9525741268224334, places = 5) self.assertAlmostEqual(out["four"], 0.9820137900379085, places = 5)
def testProbit(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.probit: input} """) self.assertAlmostEqual(engine.action(2.2), 0.9860965524865013, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.probit: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 0.841344746068543 , places = 5) self.assertAlmostEqual(out[1], 0.9772498680518207, places = 5) self.assertAlmostEqual(out[2], 0.9986501019683699, places = 5) self.assertAlmostEqual(out[3], 0.9999683287581669, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.probit: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.841344746068543 , places = 5) self.assertAlmostEqual(out["two"], 0.9772498680518207, places = 5) self.assertAlmostEqual(out["three"], 0.9986501019683699, places = 5) self.assertAlmostEqual(out["four"], 0.9999683287581669, places = 5)
def testReLu(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.relu: input} """) self.assertAlmostEqual(engine.action(2.2), 2.2, places = 1) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.relu: input} """) out = engine.action([-1, -2, 3, 4]) self.assertAlmostEqual(out[0], 0.0, places = 5) self.assertAlmostEqual(out[1], 0.0, places = 5) self.assertAlmostEqual(out[2], 3.0, places = 5) self.assertAlmostEqual(out[3], 4.0, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.relu: input} """) out = engine.action({"one": -1, "two": -2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.0, places = 5) self.assertAlmostEqual(out["two"], 0.0, places = 5) self.assertAlmostEqual(out["three"], 3.0, places = 5) self.assertAlmostEqual(out["four"], 4.0, places = 5)
def testtanh(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.tanh: input} """) self.assertAlmostEqual(engine.action(2.2), 0.9757431300314515, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.tanh: input} """) out = engine.action([-1, -2, 3, 4]) self.assertAlmostEqual(out[0], -0.761594155956, places = 5) self.assertAlmostEqual(out[1], -0.964027580076, places = 5) self.assertAlmostEqual(out[2], 0.995054753687 , places = 5) self.assertAlmostEqual(out[3], 0.999329299739 , places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.tanh: input} """) out = engine.action({"one": -1, "two": -2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], -0.761594155956, places = 5) self.assertAlmostEqual(out["two"], -0.964027580076, places = 5) self.assertAlmostEqual(out["three"], 0.995054753687 , places = 5) self.assertAlmostEqual(out["four"], 0.999329299739 , places = 5)
def testfindAll(self): engine, = PFAEngine.fromYaml(""" input: string output: {type: array, items: string} action: - {re.findall: [input, [ab]]} """) self.assertEqual(engine.action("abcabcabc"), ["ab","ab", "ab"]) self.assertEqual(engine.action("88cabcc"), ["ab"]) # check non-ascii string input # engine, = PFAEngine.fromYaml(""" #input: string #output: {type: array, items: string} #action: # - {re.findall: [input, [猫机+猫]]} #""") # self.assertEqual(engine.action("猫机猫oooo猫机机猫ppp猫机机机猫bbbb猫机aaaa猫机机"), ["猫机猫" ,"猫机机猫","猫机机机猫"]) # check byte input engine, = PFAEngine.fromYaml(""" input: bytes output: {type: array, items: bytes} action: - re.findall: [input, {bytes.encodeUtf8: {string: "ab+"}}] """) self.assertEqual(engine.action("xyz"), []) self.assertEqual(engine.action("abc机机abcabc"), ["ab", "ab","ab"] )
def testLoglog(self): engine, = PFAEngine.fromYaml(""" input: double output: double action: {m.link.loglog: input} """) self.assertAlmostEqual(engine.action(2.2), 1.203611803484212E-4, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: array, items: double} output: {type: array, items: double} action: {m.link.loglog: input} """) out = engine.action([1, 2, 3, 4]) self.assertAlmostEqual(out[0], 0.06598803584531254 , places = 5) self.assertAlmostEqual(out[1], 6.179789893310934E-4 , places = 5) self.assertAlmostEqual(out[2], 1.8921786948382924E-9 , places = 5) self.assertAlmostEqual(out[3], 1.9423376049564073E-24, places = 5) engine, = PFAEngine.fromYaml(""" input: {type: map, values: double} output: {type: map, values: double} action: {m.link.loglog: input} """) out = engine.action({"one": 1, "two": 2, "three": 3, "four": 4}) self.assertAlmostEqual(out["one"], 0.06598803584531254 , places = 5) self.assertAlmostEqual(out["two"], 6.179789893310934E-4 , places = 5) self.assertAlmostEqual(out["three"], 1.8921786948382924E-9 , places = 5) self.assertAlmostEqual(out["four"], 1.9423376049564073E-24, places = 5)
def testSubMaps(self): engine, = PFAEngine.fromYaml(''' input: "null" output: {type: map, values: double} action: la.sub: - type: {type: map, values: double} value: {one: 1, two: 2, three: 3, four: 4, five: 5} - type: {type: map, values: double} value: {one: 101, two: 102, three: 103, four: 104, five: 105, six: 999} ''') self.assertAlmostEqual(self.chi2Vector(engine.action(None), {"one": -100.0, "two": -100.0, "three": -100.0, "four": -100.0, "five": -100.0, "six": -999.0}), 0.0, places=2) engine, = PFAEngine.fromYaml(''' input: "null" output: {type: map, values: {type: map, values: double}} action: la.sub: - type: {type: map, values: {type: map, values: double}} value: {uno: {one: 1, two: 2, three: 3}, dos: {one: 4, two: 5, three: 6}, tres: {one: 7, two: 8, three: 9}} - type: {type: map, values: {type: map, values: double}} value: {uno: {one: 101, two: 102, three: 103}, dos: {one: 104, two: 105, three: 106}, tres: {one: 107, two: 108, three: 109, four: 999.0}} ''') self.assertAlmostEqual(self.chi2(engine.action(None), {"uno": {"one": -100.0, "two": -100.0, "three": -100.0, "four": 0.0}, "dos": {"one": -100.0, "two": -100.0, "three": -100.0, "four": 0.0}, "tres": {"one": -100.0, "two": -100.0, "three": -100.0, "four": -999.0}}), 0.0, places=2)
def testCartMustBuildNumericalCategorical(self): random.seed(12345) numpy.seterr(divide="ignore", invalid="ignore") dataset = Dataset.fromIterable(((x, y, c) for (x, y, z, a, b, c) in TestProducerCart.data()), 100000, ("x", "y", "c")) tree = TreeNode.fromWholeDataset(dataset, "c") tree.splitMaxDepth(2) doc = tree.pfaDocument({"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]}, "TreeNode") # look(doc, maxDepth=8) self.assertEqual(doc["cells"]["tree"]["init"]["field"], "x") self.assertAlmostEqual(doc["cells"]["tree"]["init"]["value"], 4.00, places=2) self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["field"], "y") self.assertAlmostEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["value"], 6.00, places=2) self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["pass"]["string"], "C3") self.assertEqual(doc["cells"]["tree"]["init"]["pass"]["TreeNode"]["fail"]["string"], "C6") self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["field"], "y") self.assertAlmostEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["value"], 2.00, places=2) self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["pass"]["string"], "C0") self.assertEqual(doc["cells"]["tree"]["init"]["fail"]["TreeNode"]["fail"]["string"], "C0") engine, = PFAEngine.fromJson(doc) self.assertEqual(engine.action({"x": 2.0, "y": 3.0}), "C3") self.assertEqual(engine.action({"x": 2.0, "y": 8.0}), "C6") self.assertEqual(engine.action({"x": 7.0, "y": 1.0}), "C0") self.assertEqual(engine.action({"x": 7.0, "y": 5.0}), "C0") doc = tree.pfaDocument( {"type": "record", "name": "Datum", "fields": [{"name": "x", "type": "double"}, {"name": "y", "type": "double"}]}, "TreeNode", nodeScores=True, datasetSize=True, predictandDistribution=True, predictandUnique=True, entropy=True, gain=True) # look(doc, maxDepth=8) engine, = PFAEngine.fromJson(doc)
def bad(): PFAEngine.fromYaml(''' input: ["null", "null"] output: "null" action: - {impute.defaultOnNull: [input, "null"]} ''')
def testAdditionIntOverflows(self): engine, = PFAEngine.fromYaml(''' input: int output: int action: - {+: [input, 10]} ''') self.assertEqual(engine.action(2147483637), 2147483647) self.assertRaises(PFARuntimeException, lambda: engine.action(2147483638)) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {+: [input, -10]} ''') self.assertEqual(engine.action(-2147483638), -2147483648) self.assertRaises(PFARuntimeException, lambda: engine.action(-2147483639))
def testSqrt(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {m.sqrt: input} ''') self.assertEqual(str(engine.action(-1)), "nan") self.assertAlmostEqual(engine.action(0), 0.00, places=2) self.assertAlmostEqual(engine.action(0.5), 0.71, places=2) self.assertAlmostEqual(engine.action(22.5), 4.74, places=2)
def testProvideConstants(self): engine, = PFAEngine.fromYaml(''' input: "null" output: double action: - {m.pi: []} ''') self.assertAlmostEqual(engine.action(None), 3.141592653589793, places=14) engine, = PFAEngine.fromYaml(''' input: "null" output: double action: - {m.e: []} ''') self.assertAlmostEqual(engine.action(None), 2.718281828459045, places=14)
def testSimpleTree(self): engine, = PFAEngine.fromPmml(''' <PMML version="4.2"> <Header copyright=""/> <DataDictionary> <DataField name="x" optype="continuous" dataType="double" /> <DataField name="y" optype="continuous" dataType="integer" /> <DataField name="z" optype="categorical" dataType="string" /> </DataDictionary> <TreeModel functionName="categorical" splitCharacteristic="binarySplit"> <Node> <True/> <Node> <SimplePredicate field="x" operator="lessThan" value="1"/> <Node score="leaf-1"> <SimplePredicate field="z" operator="equal" value="hello"/> </Node> <Node score="leaf-2"> <SimplePredicate field="z" operator="notEqual" value="hello"/> </Node> </Node> <Node> <SimplePredicate field="x" operator="greaterOrEqual" value="1"/> <Node score="leaf-3"> <SimplePredicate field="z" operator="equal" value="hello"/> </Node> <Node score="leaf-4"> <SimplePredicate field="z" operator="notEqual" value="hello"/> </Node> </Node> </Node> </TreeModel> </PMML> ''') self.assertEqual(engine.action({ "x": 0.9, "y": 0, "z": "hello" }), "leaf-1") self.assertEqual(engine.action({ "x": 0.9, "y": 0, "z": "goodbye" }), "leaf-2") self.assertEqual(engine.action({ "x": 1.1, "y": 0, "z": "hello" }), "leaf-3") self.assertEqual(engine.action({ "x": 1.1, "y": 0, "z": "goodbye" }), "leaf-4")
def testDoReplaceAll(self): engine, = PFAEngine.fromYaml(''' input: string output: string action: - {s.replaceall: [input, [ey], [EY]]} ''') self.assertEqual(engine.action("hey"), "hEY") self.assertEqual(engine.action("hey hey hey"), "hEY hEY hEY") self.assertEqual(engine.action("abc"), "abc") self.assertEqual(engine.action("yeh yeh yeh"), "yeh yeh yeh")
def testDoIndex(self): engine, = PFAEngine.fromYaml(''' input: string output: int action: - {s.index: [input, [ack!]]} ''') self.assertEqual(engine.action("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), -1) self.assertEqual(engine.action("ack! ack! ack!"), 0) self.assertEqual(engine.action("adfasdfadfack!asdfasdf ackasdfadfd! ack!asdfadsf"), 10) self.assertEqual(engine.action("adfasdfadack!asdfasdfasdf"), 9)
def testDoEndswith(self): engine, = PFAEngine.fromYaml(''' input: string output: boolean action: - {s.endswith: [input, [ack!]]} ''') self.assertFalse(engine.action("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) self.assertTrue(engine.action("ack! ack! ack!")) self.assertTrue(engine.action("adfasdfadfack!asdfasdf ackasdfadfd! ack!asdfadsfack!")) self.assertFalse(engine.action("adfasdfadack!asdfasdfasdf"))
def testLong(self): engine1, = PFAEngine.fromYaml(''' input: "null" output: long randseed: 12345 action: {rand.long: []} ''') self.assertEqual(engine1.action(None), 4292285838037326215) self.assertEqual(engine1.action(None), 6551146165133617474) self.assertEqual(engine1.action(None), -5650950641291792112) engine2, = PFAEngine.fromYaml(''' input: "null" output: long randseed: 12345 action: {rand.long: [5, 10]} ''') self.assertEqual(engine2.action(None), 7) self.assertEqual(engine2.action(None), 5) self.assertEqual(engine2.action(None), 9)
def testHandleSubtractionIntOverflows(self): engine, = PFAEngine.fromYaml(''' input: int output: int action: - {-: [-10, input]} ''') self.assertEqual(engine.action(2147483638), -2147483648) self.assertRaises(PFARuntimeException, lambda: engine.action(2147483639)) engine, = PFAEngine.fromYaml(''' input: int output: int action: - {-: [10, input]} ''') self.assertEqual(engine.action(-2147483637), 2147483647) self.assertRaises(PFARuntimeException, lambda: engine.action(-2147483638))
def testIsNan(self): floatEngine, = PFAEngine.fromYaml(''' input: float output: boolean action: {impute.isnan: input} ''') self.assertFalse(floatEngine.action(123.4)) self.assertTrue(floatEngine.action(float("nan"))) self.assertFalse(floatEngine.action(float("inf"))) self.assertFalse(floatEngine.action(float("-inf"))) doubleEngine, = PFAEngine.fromYaml(''' input: double output: boolean action: {impute.isnan: input} ''') self.assertFalse(doubleEngine.action(123.4)) self.assertTrue(doubleEngine.action(float("nan"))) self.assertFalse(doubleEngine.action(float("inf"))) self.assertFalse(doubleEngine.action(float("-inf")))
def testHandleNegativeLongOverflows(self): engine, = PFAEngine.fromYaml(''' input: long output: long action: - {u-: [input]} ''') self.assertEqual(engine.action(9223372036854775807), -9223372036854775807) self.assertRaises(PFARuntimeException, lambda: engine.action(-9223372036854775808))
def testDefaultOnNonNum(self): floatEngine, = PFAEngine.fromYaml(''' input: float output: float action: {impute.defaultOnNonNum: [input, {float: 999.0}]} ''') self.assertEqual(floatEngine.action(123.4), 123.4) self.assertEqual(floatEngine.action(float("nan")), 999.0) self.assertEqual(floatEngine.action(float("inf")), 999.0) self.assertEqual(floatEngine.action(float("-inf")), 999.0) doubleEngine, = PFAEngine.fromYaml(''' input: double output: double action: {impute.defaultOnNonNum: [input, 999.0]} ''') self.assertEqual(doubleEngine.action(123.4), 123.4) self.assertEqual(doubleEngine.action(float("nan")), 999.0) self.assertEqual(doubleEngine.action(float("inf")), 999.0) self.assertEqual(doubleEngine.action(float("-inf")), 999.0)
def testInt(self): engine1, = PFAEngine.fromYaml(''' input: "null" output: int randseed: 12345 action: {rand.int: []} ''') self.assertEqual(engine1.action(None), -358114921) self.assertEqual(engine1.action(None), -2103807398) self.assertEqual(engine1.action(None), 1396751321) engine2, = PFAEngine.fromYaml(''' input: "null" output: int randseed: 12345 action: {rand.int: [5, 10]} ''') self.assertEqual(engine2.action(None), 7) self.assertEqual(engine2.action(None), 5) self.assertEqual(engine2.action(None), 9)
def testEncodeAscii(self): engine, = PFAEngine.fromYaml(''' input: string output: bytes action: bytes.encodeAscii: input ''') self.assertEqual(engine.action("hello"), struct.pack("bbbbb", 104, 101, 108, 108, 111)) self.assertRaises(PFARuntimeException, lambda: engine.action("hel\x81o"))
def testErrorOnNull(self): engine, = PFAEngine.fromYaml(''' input: [int, "null"] output: int action: - {impute.errorOnNull: [input]} ''') self.assertEqual(engine.action(3), 3) self.assertEqual(engine.action(12), 12) self.assertRaises(PFARuntimeException, lambda: engine.action(None)) self.assertEqual(engine.action(5), 5)
def testCheckUtf16le(self): engine, = PFAEngine.fromYaml(''' input: bytes output: boolean action: bytes.isUtf16le: input ''') self.assertEqual( engine.action( struct.pack("bbbbbbbbbb", 104, 0, 101, 0, 108, 0, 108, 0, 111, 0)), True)
def testEncodeUtf16(self): engine, = PFAEngine.fromYaml(''' input: string output: bytes action: bytes.encodeUtf16: input ''') self.assertEqual( engine.action("hello"), struct.pack("bbbbbbbbbbbb", -1, -2, 104, 0, 101, 0, 108, 0, 108, 0, 111, 0))
def testIn(self): engine, = PFAEngine.fromYaml(''' input: int output: boolean action: map.in: - {map.toset: {value: [1, 2, 3, 4, 5], type: {type: array, items: int}}} - input ''') self.assertTrue(engine.action(2)) self.assertFalse(engine.action(0))
def testDecodeUtf16le(self): engine, = PFAEngine.fromYaml(''' input: bytes output: string action: bytes.decodeUtf16le: input ''') self.assertEqual( engine.action( struct.pack("bbbbbbbbbb", 104, 0, 101, 0, 108, 0, 108, 0, 111, 0)), "hello")
def testDecodeLatin1(self): engine, = PFAEngine.fromYaml(''' input: bytes output: string action: bytes.decodeLatin1: input ''') self.assertEqual( engine.action(struct.pack("bbbbb", 104, 101, 108, 108, 111)), "hello") self.assertEqual(engine.action('hello'), "hello")
def testSymDiff(self): engine, = PFAEngine.fromYaml(''' input: "null" output: {type: array, items: int} action: map.fromset: map.symdiff: - {map.toset: {value: [1, 2, 3, 4, 5], type: {type: array, items: int}}} - {map.toset: {value: [4, 5, 6, 7, 8], type: {type: array, items: int}}} ''') self.assertEqual(set(engine.action(None)), set([1, 2, 3, 6, 7, 8]))
def testToString(self): engine, = PFAEngine.fromYaml(''' input: {type: enum, name: Test, symbols: ["A", "B", "C"]} output: string action: enum.toString: input ''') self.assertEqual(engine.action("A"), "A") self.assertEqual(engine.action("B"), "B") self.assertEqual(engine.action("C"), "C") self.assertRaises(AvroException, lambda: engine.action("D"))
def testSignum(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {m.signum: input} ''') self.assertEqual(engine.action(-3.2), -1) self.assertEqual(engine.action(0), 0) self.assertEqual(engine.action(3.2), 1) self.assertEqual(engine.action(1.0), 1)
def testrIndex(self): engine, = PFAEngine.fromYaml(""" input: string output: {type: array, items: int} action: - {re.rindex: [input, [ab(c|d)*]]} """) self.assertEqual(engine.action("abcccdc"), [0,7]) self.assertEqual(engine.action("abddddd"), [0,7]) self.assertEqual(engine.action("XKASGJ8"), []) engine, = PFAEngine.fromYaml(""" input: string output: {type: array, items: int} action: - {re.rindex: [input, [dog]]} """) self.assertEqual(engine.action("999dogggggg"), [3,6]) self.assertEqual(engine.action("cat"), []) self.assertEqual(engine.action("catdogpppdog"), [9,12]) # check non-ascii string input # engine, = PFAEngine.fromYaml(""" #input: string #output: {type: array, items: int} #action: # - {re.rindex: [input, [对讲机(讲|机)*]]} #""") # self.assertEqual(engine.action("abcccdc"), []) # self.assertEqual(engine.action("xyzzzz对讲机机abcc对讲机机mmmmm对讲机机aa"), [23,27]) # check byte input engine, = PFAEngine.fromYaml(""" input: bytes output: {type: array, items: int} action: - re.rindex: [input, {bytes.encodeUtf8: {string: "对讲机(讲|机)*"}}] """) self.assertEqual(engine.action("abcccdc"), []) self.assertEqual(engine.action("xyzzzz对讲机机abcc对讲机机mmmmm对讲机机aa"), [39,51])
def testHistogram2(self): engine, = PFAEngine.fromYaml(''' input: "null" output: HistogramItem randseed: 12345 cells: hist: type: type: array items: type: record name: HistogramItem fields: - {name: label, type: string} - {name: prob, type: double} init: - {label: A, prob: 3.3} - {label: B, prob: 2.2} - {label: C, prob: 5.5} - {label: D, prob: 0.0} - {label: E, prob: 1.1} - {label: F, prob: 8.8} action: {rand.histogram: {cell: hist}} ''') results = [engine.action(None) for i in range(0, 10000)] self.assertAlmostEqual(sum(1 for x in results if x["label"] == "A") / 10000.0, 0.15789473684210525, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "B") / 10000.0, 0.10526315789473686, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "C") / 10000.0, 0.26315789473684215, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "D") / 10000.0, 0.0, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "E") / 10000.0, 0.05263157894736843, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "F") / 10000.0, 0.42105263157894746, places=2) self.assertAlmostEqual(sum(1 for x in results if x["label"] == "G") / 10000.0, 0.0, places=2)
def testErfcFcn(self): engine, = PFAEngine.fromYaml(''' input: double output: double action: - {m.special.erfc: input} ''') self.assertAlmostEqual(engine.action(-22.5), 2.00, places=3) self.assertAlmostEqual(engine.action(-0.5), 1.52, places=3) self.assertAlmostEqual(engine.action(0), 1.00, places=3) self.assertAlmostEqual(engine.action(0.5), 0.4795, places=3) self.assertAlmostEqual(engine.action(22.5), 0.00, places=3)
def testDoUpper(self): engine, = PFAEngine.fromYaml(''' input: string output: string action: - {s.upper: [input]} ''') self.assertEqual(engine.action("hey"), "HEY") self.assertEqual(engine.action("Hey"), "HEY") self.assertEqual(engine.action("HEY"), "HEY") self.assertEqual(engine.action("hEy"), "HEY") self.assertEqual(engine.action("heY"), "HEY")
def testAccumulateACovarianceAndUseThatToUpdateAPCAMatrixUsingArrays(self): engine, = PFAEngine.fromYaml(''' input: {type: array, items: double} output: {type: array, items: double} cells: state: type: type: record name: State fields: - {name: count, type: double} - {name: mean, type: {type: array, items: double}} - {name: covariance, type: {type: array, items: {type: array, items: double}}} init: count: 0 mean: [0, 0, 0, 0, 0] covariance: [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]] action: - let: mat: la.truncate: - la.eigenBasis: - attr: cell: state to: params: [{state: State}] ret: State do: stat.sample.updateCovariance: - input - 1.0 - state path: [[covariance]] - 2 - let: vec: a.mapWithIndex: - input - params: [{i: int}, {x: double}] ret: double do: {"-": [x, {cell: state, path: [[mean], i]}]} - if: {">=": [{cell: state, path: [[count]]}, 2]} then: {la.dot: [mat, vec]} else: {type: {type: array, items: double}, value: []} ''') engine.action([23, 56, 12, 34, 72]) engine.action([52, 61, 12, 71, 91]) engine.action([15, 12, 89, 23, 48]) self.assertAlmostEqual(self.chi2Vector( engine.action([16, 27, 36, 84, 52]), [-0.038, -1.601]), 0.0, places=2)
def testHourOfDay(self): engine, = PFAEngine.fromYaml(""" input: double output: int action: - {time.hourOfDay: [input, {string: ""}]} """) self.assertEqual(engine.action(0), 0) self.assertEqual(engine.action(1425508527), 22) self.assertEqual(engine.action(1.425508527E9), 22) self.assertEqual(engine.action(1425508527.52482), 22) engine2, = PFAEngine.fromYaml(""" input: string output: int action: - {time.hourOfDay: [0, input]} """) self.assertEqual(engine2.action("Europe/Paris"), 1) self.assertEqual(engine2.action("Etc/UTC"), 0) self.assertEqual(engine2.action("Atlantic/Azores"), 23)
def testDisjoint(self): engine, = PFAEngine.fromYaml(''' input: {type: array, items: int} output: boolean action: map.disjoint: - {map.toset: input} - {map.toset: {value: [1, 2, 3, 4, 5], type: {type: array, items: int}}} ''') self.assertFalse(engine.action([1, 2, 3])) self.assertFalse(engine.action([1, 2, 3, 999])) self.assertTrue(engine.action([888, 999]))