def greedyNestedStructuralTest(): pattern = Pattern( AndOperator( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b")), SeqOperator(PrimitiveEventStructure("AVID", "c"), PrimitiveEventStructure("BIDU", "d"))), AndCondition( BinaryCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x, y: x > y), BinaryCondition(Variable("d", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y: x > y), EqCondition(Variable("a", lambda x: x["Date"]), 200802010900), EqCondition(Variable("b", lambda x: x["Date"]), 200802010900), EqCondition(Variable("c", lambda x: x["Date"]), 200802010900), EqCondition(Variable("d", lambda x: x["Date"]), 200802010900)), timedelta(minutes=3)) selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0], [0.9457796098355941, 1.0, 0.15989723367389616, 1.0], [1.0, 0.15989723367389616, 1.0, 0.9992557393942864], [1.0, 1.0, 0.9992557393942864, 1.0]] arrivalRates = [ 0.016597077244258872, 0.01454418928322895, 0.013917884481558803, 0.012421711899791231 ] pattern.set_statistics({ StatisticsTypes.ARRIVAL_RATES: arrivalRates, StatisticsTypes.SELECTIVITY_MATRIX: selectivityMatrix }) eval_params = TreeBasedEvaluationMechanismParameters( optimizer_params=OptimizerParameters( opt_type=OptimizerTypes.TRIVIAL_OPTIMIZER, tree_plan_params=TreePlanBuilderParameters( TreePlanBuilderTypes.GREEDY_LEFT_DEEP_TREE)), storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS. storage_params) expected_result = ('And', ('Seq', 'b', 'a'), ('Seq', 'd', 'c')) runStructuralTest('greedyNestedStructuralTest', [pattern], expected_result, eval_mechanism_params=eval_params)
def KC_AND(createTestFile=False): """ KC(And([a, b, c])) """ pattern = Pattern( KleeneClosureOperator( AndOperator( PrimitiveEventStructure("GOOG", "a"), PrimitiveEventStructure("GOOG", "b"), PrimitiveEventStructure("GOOG", "c") ), min_size=1, max_size=3 ), AndCondition( SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])), SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"])), ), timedelta(minutes=3) ) runTest("KC_AND_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def KC_AND_IndexCondition_02(createTestFile=False): """ KC(And([a, b])) """ pattern = Pattern( KleeneClosureOperator( AndOperator( PrimitiveEventStructure("GOOG", "a"), PrimitiveEventStructure("GOOG", "b") ), min_size=1, max_size=3 ), AndCondition( SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])), KCIndexCondition(names={'a', 'b'}, getattr_func=lambda x: x["Peak Price"], relation_op=lambda x, y: x < y, offset=2), ), timedelta(minutes=3) ) runTest("KC_AND_IndexCondition_02_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def threePatternsTestFullSharing(createTestFile=False): pattern1 = Pattern( AndOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("GOOG", "c")), AndCondition( SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])), SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]))), timedelta(minutes=1)) pattern2 = Pattern( SeqOperator(PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"), PrimitiveEventStructure("MSFT", "c"), PrimitiveEventStructure("DRIV", "d"), PrimitiveEventStructure("MSFT", "e")), AndCondition( SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])), SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"])), SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]), Variable("d", lambda x: x["Peak Price"])), SmallerThanCondition(Variable("d", lambda x: x["Peak Price"]), Variable("e", lambda x: x["Peak Price"]))), timedelta(minutes=10)) pattern3 = Pattern( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("GOOG", "c")), AndCondition( GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"])), GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]))), timedelta(minutes=5)) runMultiTest("ThreePatternTestFullSharing", [pattern1, pattern2, pattern3], createTestFile, subtree_sharing_eval_mechanism_params, expected_file_name="ThreePatternTest")
def hierarchyPatternSearchTest(createTestFile=False): """ The following pattern is looking for Amazon < Apple < Google cases in one minute windows. PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c) WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice WITHIN 1 minute """ hierarchyPattern = Pattern( AndOperator( [QItem("AMZN", "a"), QItem("AAPL", "b"), QItem("GOOG", "c")]), AndFormula( SmallerThanFormula(IdentifierTerm("a", lambda x: x["Peak Price"]), IdentifierTerm("b", lambda x: x["Peak Price"])), SmallerThanFormula(IdentifierTerm("b", lambda x: x["Peak Price"]), IdentifierTerm("c", lambda x: x["Peak Price"]))), timedelta(minutes=1)) runTest('hierarchy', [hierarchyPattern], createTestFile)
def hierarchyPatternSearchTest(createTestFile=False): """ The following pattern is looking for Amazon < Apple < Google cases in one minute windows. PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c) WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice WITHIN 1 minute """ hierarchyPattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AAPL", "b"), PrimitiveEventStructure("GOOG", "c")), AndCondition( BinaryCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y), BinaryCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y) ), timedelta(minutes=1) ) runTest('hierarchy', [hierarchyPattern], createTestFile)
def googleAmazonLowPatternSearchTest(createTestFile=False, eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS, test_name = "googleAmazonLow"): """ This pattern is looking for low prices of Amazon and Google at the same minute. PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g) WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525 WITHIN 1 minute """ googleAmazonLowPattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")), AndCondition( SimpleCondition(Variable("a", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 73), SimpleCondition(Variable("g", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 525) ), timedelta(minutes=1) ) runTest(test_name, [googleAmazonLowPattern], createTestFile, eval_mechanism_params)
def structuralTest6(): """ Seq([a, Seq([ b, And([c, d]), e])]) """ structural_test_pattern = Pattern( SeqOperator( PrimitiveEventStructure("GOOG", "a"), SeqOperator( PrimitiveEventStructure("GOOG", "b"), AndOperator( PrimitiveEventStructure("GOOG", "c"), PrimitiveEventStructure("GOOG", "d") ), PrimitiveEventStructure("GOOG", "e") ), ), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=3) ) expected_result = ('Seq', 'a', ('Seq', ('Seq', 'b', ('And', 'c', 'd')), 'e')) runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)
def sortedStorageTest(createTestFile=False): pattern = Pattern( AndOperator(PrimitiveEventStructure("DRIV", "a"), PrimitiveEventStructure("MSFT", "b"), PrimitiveEventStructure("CBRL", "c")), AndCondition( GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"])), GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"])), ), timedelta(minutes=360), ) storage_params = TreeStorageParameters(True, clean_up_interval=500) eval_params = TreeBasedEvaluationMechanismParameters( DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params, storage_params) runTest("sortedStorageTest", [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def basicNestedTest(createTestFile=False): pattern = Pattern( AndOperator( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b")), SeqOperator(PrimitiveEventStructure("AVID", "c"), PrimitiveEventStructure("BIDU", "d"))), AndCondition( BinaryCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x, y: x > y), BinaryCondition(Variable("d", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y: x > y), EqCondition(Variable("a", lambda x: x["Date"]), 200802010900), EqCondition(Variable("b", lambda x: x["Date"]), 200802010900), EqCondition(Variable("c", lambda x: x["Date"]), 200802010900), EqCondition(Variable("d", lambda x: x["Date"]), 200802010900)), timedelta(minutes=5)) expected_result = ('And', ('Seq', 'a', 'b'), ('Seq', 'c', 'd')) runStructuralTest('basicNestedStructuralTest', [pattern], expected_result) runTest("basicNested", [pattern], createTestFile)
def nonsensePatternSearchTest(createTestFile=False): """ This pattern is looking for something that does not make sense. PATTERN AND(AmazonStockPriceUpdate a, AvidStockPriceUpdate b, AppleStockPriceUpdate c) WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < a.PeakPrice """ nonsensePattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AVID", "b"), PrimitiveEventStructure("AAPL", "c")), AndCondition( BinaryCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y), BinaryCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y), BinaryCondition(Variable("c", lambda x: x["Peak Price"]), Variable("a", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y) ), timedelta(minutes=1) ) runTest('nonsense', [nonsensePattern], createTestFile)
def sortedStorageTest(createTestFile=False): pattern = Pattern( AndOperator(PrimitiveEventStructure("DRIV", "a"), PrimitiveEventStructure("MSFT", "b"), PrimitiveEventStructure("CBRL", "c")), AndCondition( GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"])), GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"])), ), timedelta(minutes=360), ) storage_params = TreeStorageParameters(True, clean_up_interval=500) eval_params = TreeBasedEvaluationMechanismParameters( optimizer_params=StatisticsDeviationAwareOptimizerParameters( tree_plan_params=TreePlanBuilderParameters()), storage_params=storage_params) runTest("sortedStorageTest", [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def sortedStorageTest(createTestFile=False): pattern = Pattern( AndOperator( [QItem("DRIV", "a"), QItem("MSFT", "b"), QItem("CBRL", "c")]), AndFormula( GreaterThanFormula( IdentifierTerm("a", lambda x: x["Opening Price"]), IdentifierTerm("b", lambda x: x["Opening Price"])), GreaterThanFormula( IdentifierTerm("b", lambda x: x["Opening Price"]), IdentifierTerm("c", lambda x: x["Opening Price"])), ), timedelta.max, ) runTest( "sortedStorageTest", [pattern], createTestFile, eval_mechanism_type=EvaluationMechanismTypes.TRIVIAL_LEFT_DEEP_TREE, events=nasdaqEventStream, )
def nonsensePatternSearchTest(createTestFile=False): """ This pattern is looking for something that does not make sense. PATTERN AND(AmazonStockPriceUpdate a, AvidStockPriceUpdate b, AppleStockPriceUpdate c) WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < a.PeakPrice """ nonsensePattern = Pattern( AndOperator( [QItem("AMZN", "a"), QItem("AVID", "b"), QItem("AAPL", "c")]), AndFormula( SmallerThanFormula(IdentifierTerm("a", lambda x: x["Peak Price"]), IdentifierTerm("b", lambda x: x["Peak Price"])), AndFormula( SmallerThanFormula( IdentifierTerm("b", lambda x: x["Peak Price"]), IdentifierTerm("c", lambda x: x["Peak Price"])), SmallerThanFormula( IdentifierTerm("c", lambda x: x["Peak Price"]), IdentifierTerm("a", lambda x: x["Peak Price"])))), timedelta(minutes=1)) runTest('nonsense', [nonsensePattern], createTestFile)
def hierarchyPatternSearchTest(createTestFile=False, eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS, test_name = "hierarchy"): """ The following pattern is looking for Amazon < Apple < Google cases in one minute windows. PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c) WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice WITHIN 1 minute """ hierarchyPattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AAPL", "b"), PrimitiveEventStructure("GOOG", "c")), AndCondition( BinaryCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y), BinaryCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), relation_op=lambda x, y: x < y) ), timedelta(minutes=1) ) runTest(test_name, [hierarchyPattern], createTestFile, eval_mechanism_params)
def structuralTest1(): """ Seq([a, KC(And([KC(d), KC(Seq([e, f]))]))]) """ structural_test_pattern = Pattern( SeqOperator(PrimitiveEventStructure("GOOG", "a"), KleeneClosureOperator( AndOperator(PrimitiveEventStructure("GOOG", "b"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "c"), min_size=1, max_size=5), KleeneClosureOperator(SeqOperator(PrimitiveEventStructure("GOOG", "d"), PrimitiveEventStructure("GOOG", "e")), min_size=1, max_size=5) ), min_size=1, max_size=5, )), AndCondition( SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), SimpleCondition(Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135) ), timedelta(minutes=3) ) expected_result = ('Seq', 'a', ('KC', ('And', ('And', 'b', ('KC', 'c')), ('KC', ('Seq', 'd', 'e'))))) runStructuralTest('structuralTest1', [structural_test_pattern], expected_result)
def threePatternsTest(createTestFile=False): pattern1 = Pattern( AndOperator( PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), ), AndCondition( SmallerThanCondition( Variable("a", lambda x: x["Peak Price"] - x["Opening Price"]), Variable("b", lambda x: x["Peak Price"] - x["Opening Price"]), ), ), timedelta(minutes=1), confidence=0.96) pattern2 = Pattern( SeqOperator( PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"), PrimitiveEventStructure("MSFT", "c"), PrimitiveEventStructure("DRIV", "d"), PrimitiveEventStructure("MSFT", "e"), ), AndCondition( SmallerThanCondition( Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), ), SmallerThanCondition( Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), ), SmallerThanCondition( Variable("c", lambda x: x["Peak Price"]), Variable("d", lambda x: x["Peak Price"]), ), SmallerThanCondition( Variable("d", lambda x: x["Peak Price"]), Variable("e", lambda x: x["Peak Price"]), ), ), timedelta(minutes=10), confidence=0.6, ) pattern3 = Pattern( SeqOperator( PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("GOOG", "c"), ), AndCondition( GreaterThanCondition( Variable("a", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), ), GreaterThanCondition( Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), ), ), timedelta(minutes=5), confidence=0.6, ) runMultiTest("ThreePatternTestProb", [pattern1, pattern2, pattern3], createTestFile, eventStream=nasdaqEventStreamP)