def dpLdPatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"), PrimitiveEventStructure("ORLY", "c"), PrimitiveEventStructure("CBRL", "d")), AndCondition( SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])), SimpleCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), Variable("d", lambda x: x["Peak Price"]), relation_op=lambda x, y, z: x < y < z)), timedelta(minutes=3)) selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0], [0.9457796098355941, 1.0, 0.15989723367389616, 1.0], [1.0, 0.15989723367389616, 1.0, 0.9992557393942864], [1.0, 1.0, 0.9992557393942864, 1.0]] arrivalRates = [ 0.016597077244258872, 0.01454418928322895, 0.013917884481558803, 0.012421711899791231 ] pattern.set_statistics( StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES, (selectivityMatrix, arrivalRates)) eval_params = TreeBasedEvaluationMechanismParameters( TreePlanBuilderParameters( TreePlanBuilderTypes.DYNAMIC_PROGRAMMING_LEFT_DEEP_TREE), DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params) runTest('dpLd1', [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def MinMax_2_TestKleeneClosure(createTestFile=False): pattern = Pattern( SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a"), min_size=4, max_size=5)), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 0), timedelta(minutes=5) ) runTest("MinMax_2_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def iiRandom2PatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"), PrimitiveEventStructure("ORLY", "c"), PrimitiveEventStructure("CBRL", "d")), AndCondition( SimpleCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), Variable("d", lambda x: x["Peak Price"]), relation_op=lambda x, y, z, w: x < y < z < w)), timedelta(minutes=3)) selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0], [0.9457796098355941, 1.0, 0.15989723367389616, 1.0], [1.0, 0.15989723367389616, 1.0, 0.9992557393942864], [1.0, 1.0, 0.9992557393942864, 1.0]] arrivalRates = [ 0.016597077244258872, 0.01454418928322895, 0.013917884481558803, 0.012421711899791231 ] pattern.set_statistics( StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES, (selectivityMatrix, arrivalRates)) eval_params = TreeBasedEvaluationMechanismParameters( IterativeImprovementTreePlanBuilderParameters( DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params. cost_model_type, 20, IterativeImprovementType.CIRCLE_BASED, IterativeImprovementInitType.RANDOM), DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params) runTest('iiRandom2', [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def run_twitter_sanity_check(): """ This basic test invokes a simple pattern looking for two tweets that retweeted the same tweet. It might help finding users with common interests. PATTERN SEQ(Tweet a, Tweet b) WHERE a.Retweeted_Status_Id != None AND a.ID != b.ID AND a.Retweeted_Status_Id == b.Retweeted_Status_Id """ get_retweeted_status_function = lambda x: x[ "retweeted_status"] if "retweeted_status" in x else None pattern_retweet = Pattern( SeqOperator( PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE, "a"), PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE, "b")), AndCondition( NotEqCondition(Variable("a", lambda x: x["id"]), Variable("b", lambda x: x["id"])), SimpleCondition(Variable("a", get_retweeted_status_function), relation_op=lambda x: x is not None), EqCondition(Variable("a", get_retweeted_status_function), Variable("b", get_retweeted_status_function))), timedelta(minutes=30)) cep = CEP([pattern_retweet]) event_stream = TwitterInputStream(['corona']) try: running_time = cep.run( event_stream, FileOutputStream(os.getcwd(), "output.txt", True), TweetDataFormatter()) print("Test twitterSanityCheck result: Succeeded, Time Passed: %s" % (running_time, )) finally: event_stream.close()
def oneArgumentsearchTestKleeneClosure(createTestFile=False): pattern = Pattern( SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("AAPL", "a"), min_size=1, max_size=5)), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=5) ) runTest("oneArgumentKC", [pattern], createTestFile)
def googleAmazonLowPatternSearchTest(createTestFile=False): """ This pattern is looking for low prices of Amazon and Google at the same minute. PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g) WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525 WITHIN 1 minute """ googleAmazonLowPattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")), AndCondition( SimpleCondition(Variable("a", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 73), SimpleCondition(Variable("g", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 525) ), timedelta(minutes=1) ) runTest('googleAmazonLow', [googleAmazonLowPattern], createTestFile)
def KC_Specific_Value(createTestFile=False): pattern = Pattern( SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a"))), AndCondition( SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 0), KCValueCondition(names={'a'}, getattr_func=lambda x: x["Peak Price"], relation_op=lambda x, y: x > y, index=2, value=530.5) ), timedelta(minutes=5) ) runTest("KC_Specific_Value_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def googleAmazonLowPatternSearchTest(createTestFile=False, eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS, test_name = "googleAmazonLow"): """ This pattern is looking for low prices of Amazon and Google at the same minute. PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g) WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525 WITHIN 1 minute """ googleAmazonLowPattern = Pattern( AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")), AndCondition( SimpleCondition(Variable("a", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 73), SimpleCondition(Variable("g", lambda x: x["Peak Price"]), relation_op=lambda x: x <= 525) ), timedelta(minutes=1) ) runTest(test_name, [googleAmazonLowPattern], createTestFile, eval_mechanism_params)
def get_pattern_test(): pattern = Pattern( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("LOCM", "c")), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y, z: x > y > z), timedelta(minutes=5)) return pattern
def structuralTest2(): """ KC(a) """ structural_test_pattern = Pattern( KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a")), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=3) ) expected_result = ('KC', 'a') runStructuralTest('structuralTest2', [structural_test_pattern], expected_result)
def frequencyPatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("LOCM", "c")), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y, z: x > y > z), timedelta(minutes=5)) runTest("nonFrequency", [pattern], createTestFile)
def structuralTest3(): """ Seq([a, KC(b)]) """ structural_test_pattern = Pattern( SeqOperator( PrimitiveEventStructure("GOOG", "a"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "b")) ), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=3) ) expected_result = ('Seq', 'a', ('KC', 'b')) runStructuralTest('structuralTest3', [structural_test_pattern], expected_result)
def structuralTest1(): """ Seq([a, KC(And([KC(d), KC(Seq([e, f]))]))]) """ structural_test_pattern = Pattern( SeqOperator(PrimitiveEventStructure("GOOG", "a"), KleeneClosureOperator( AndOperator(PrimitiveEventStructure("GOOG", "b"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "c"), min_size=1, max_size=5), KleeneClosureOperator(SeqOperator(PrimitiveEventStructure("GOOG", "d"), PrimitiveEventStructure("GOOG", "e")), min_size=1, max_size=5) ), min_size=1, max_size=5, )), AndCondition( SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), SimpleCondition(Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135) ), timedelta(minutes=3) ) expected_result = ('Seq', 'a', ('KC', ('And', ('And', 'b', ('KC', 'c')), ('KC', ('Seq', 'd', 'e'))))) runStructuralTest('structuralTest1', [structural_test_pattern], expected_result)
def arrivalRatesPatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("AAPL", "a"), PrimitiveEventStructure("AMZN", "b"), PrimitiveEventStructure("LOCM", "c")), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y, z: x > y > z), timedelta(minutes=5)) pattern.set_statistics(StatisticsTypes.ARRIVAL_RATES, [0.0159, 0.0153, 0.0076]) eval_params = TreeBasedEvaluationMechanismParameters( TreePlanBuilderParameters( TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE), DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params) runTest("arrivalRates", [pattern], createTestFile, eval_params)
def nonFrequencyTailoredPatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("DRIV", "a"), PrimitiveEventStructure("MSFT", "b"), PrimitiveEventStructure("CBRL", "c")), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), Variable("b", lambda x: x["Opening Price"]), Variable("c", lambda x: x["Opening Price"]), relation_op=lambda x, y, z: x > y > z), timedelta(minutes=360)) eval_params = TreeBasedEvaluationMechanismParameters( TreePlanBuilderParameters(TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE), DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params) runTest('nonFrequencyTailored1', [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def iiRandomPatternSearchTest(createTestFile=False): pattern = Pattern( SeqOperator(PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"), PrimitiveEventStructure("ORLY", "c"), PrimitiveEventStructure("CBRL", "d")), AndCondition( SimpleCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"]), relation_op=lambda x, y, z: x < y < z), SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]), Variable("d", lambda x: x["Peak Price"]))), timedelta(minutes=3)) selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0], [0.9457796098355941, 1.0, 0.15989723367389616, 1.0], [1.0, 0.15989723367389616, 1.0, 0.9992557393942864], [1.0, 1.0, 0.9992557393942864, 1.0]] arrivalRates = [ 0.016597077244258872, 0.01454418928322895, 0.013917884481558803, 0.012421711899791231 ] pattern.set_statistics({ StatisticsTypes.SELECTIVITY_MATRIX: selectivityMatrix, StatisticsTypes.ARRIVAL_RATES: arrivalRates }) eval_params = TreeBasedEvaluationMechanismParameters( optimizer_params=StatisticsDeviationAwareOptimizerParameters( tree_plan_params=IterativeImprovementTreePlanBuilderParameters( DEFAULT_TREE_COST_MODEL, 20, IterativeImprovementType.SWAP_BASED, IterativeImprovementInitType.RANDOM), statistics_collector_params=StatisticsCollectorParameters( statistics_types=[ StatisticsTypes.ARRIVAL_RATES, StatisticsTypes.SELECTIVITY_MATRIX ])), storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS. storage_params) runTest('iiRandom1', [pattern], createTestFile, eval_mechanism_params=eval_params, events=nasdaqEventStream)
def structuralTest6(): """ Seq([a, Seq([ b, And([c, d]), e])]) """ structural_test_pattern = Pattern( SeqOperator( PrimitiveEventStructure("GOOG", "a"), SeqOperator( PrimitiveEventStructure("GOOG", "b"), AndOperator( PrimitiveEventStructure("GOOG", "c"), PrimitiveEventStructure("GOOG", "d") ), PrimitiveEventStructure("GOOG", "e") ), ), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=3) ) expected_result = ('Seq', 'a', ('Seq', ('Seq', 'b', ('And', 'c', 'd')), 'e')) runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)
def structuralTest6(): """ And([a, b, c, Seq([ d, KC(And([ e, KC(f), g ]), And([ KC(h), KC(Seq([ i, j]) ]) ]), k, l ]) """ structural_test_pattern = Pattern( AndOperator( PrimitiveEventStructure("GOOG", "a"), PrimitiveEventStructure("GOOG", "b"), PrimitiveEventStructure("GOOG", "c"), SeqOperator( PrimitiveEventStructure("GOOG", "d"), KleeneClosureOperator( AndOperator( PrimitiveEventStructure("GOOG", "e"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "f")), PrimitiveEventStructure("GOOG", "g") ) ), AndOperator( KleeneClosureOperator(PrimitiveEventStructure("GOOG", "h")), KleeneClosureOperator( SeqOperator( PrimitiveEventStructure("GOOG", "i"), PrimitiveEventStructure("GOOG", "j") ), ), ), ), PrimitiveEventStructure("GOOG", "k"), PrimitiveEventStructure("GOOG", "l") ), SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135), timedelta(minutes=3) ) expected_result = ('And', ('And', ('And', ('And', ('And', 'a', 'b'), 'c'), ('Seq', ('Seq', 'd', ('KC', ('And', ('And', 'e', ('KC', 'f')), 'g'))), ('And', ('KC', 'h'), ('KC', ('Seq', 'i', 'j'))))), 'k'), 'l') runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)