def dpLdPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SimpleCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]),
relation_op=lambda x, y, z: x < y < z)),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics(
StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES,
(selectivityMatrix, arrivalRates))
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.DYNAMIC_PROGRAMMING_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('dpLd1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def MinMax_2_TestKleeneClosure(createTestFile=False):
pattern = Pattern(
SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a"), min_size=4, max_size=5)),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 0),
timedelta(minutes=5)
)
runTest("MinMax_2_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def iiRandom2PatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]),
relation_op=lambda x, y, z, w: x < y < z < w)),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics(
StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES,
(selectivityMatrix, arrivalRates))
eval_params = TreeBasedEvaluationMechanismParameters(
IterativeImprovementTreePlanBuilderParameters(
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params.
cost_model_type, 20, IterativeImprovementType.CIRCLE_BASED,
IterativeImprovementInitType.RANDOM),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('iiRandom2', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def run_twitter_sanity_check():
"""
This basic test invokes a simple pattern looking for two tweets that retweeted the same tweet.
It might help finding users with common interests.
PATTERN SEQ(Tweet a, Tweet b)
WHERE a.Retweeted_Status_Id != None AND a.ID != b.ID AND a.Retweeted_Status_Id == b.Retweeted_Status_Id
"""
get_retweeted_status_function = lambda x: x[
"retweeted_status"] if "retweeted_status" in x else None
pattern_retweet = Pattern(
SeqOperator(
PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE,
"a"),
PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE,
"b")),
AndCondition(
NotEqCondition(Variable("a", lambda x: x["id"]),
Variable("b", lambda x: x["id"])),
SimpleCondition(Variable("a", get_retweeted_status_function),
relation_op=lambda x: x is not None),
EqCondition(Variable("a", get_retweeted_status_function),
Variable("b", get_retweeted_status_function))),
timedelta(minutes=30))
cep = CEP([pattern_retweet])
event_stream = TwitterInputStream(['corona'])
try:
running_time = cep.run(
event_stream, FileOutputStream(os.getcwd(), "output.txt", True),
TweetDataFormatter())
print("Test twitterSanityCheck result: Succeeded, Time Passed: %s" %
(running_time, ))
finally:
event_stream.close()
def oneArgumentsearchTestKleeneClosure(createTestFile=False):
pattern = Pattern(
SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("AAPL", "a"), min_size=1, max_size=5)),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=5)
)
runTest("oneArgumentKC", [pattern], createTestFile)
def googleAmazonLowPatternSearchTest(createTestFile=False):
"""
This pattern is looking for low prices of Amazon and Google at the same minute.
PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g)
WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525
WITHIN 1 minute
"""
googleAmazonLowPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 73),
SimpleCondition(Variable("g", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 525)
),
timedelta(minutes=1)
)
runTest('googleAmazonLow', [googleAmazonLowPattern], createTestFile)
def KC_Specific_Value(createTestFile=False):
pattern = Pattern(
SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a"))),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 0),
KCValueCondition(names={'a'}, getattr_func=lambda x: x["Peak Price"], relation_op=lambda x, y: x > y, index=2, value=530.5)
),
timedelta(minutes=5)
)
runTest("KC_Specific_Value_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def googleAmazonLowPatternSearchTest(createTestFile=False,
eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS,
test_name = "googleAmazonLow"):
"""
This pattern is looking for low prices of Amazon and Google at the same minute.
PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g)
WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525
WITHIN 1 minute
"""
googleAmazonLowPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 73),
SimpleCondition(Variable("g", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 525)
),
timedelta(minutes=1)
)
runTest(test_name, [googleAmazonLowPattern], createTestFile, eval_mechanism_params)
def get_pattern_test():
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LOCM", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=5))
return pattern
def structuralTest2():
"""
KC(a)
"""
structural_test_pattern = Pattern(
KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=3)
)
expected_result = ('KC', 'a')
runStructuralTest('structuralTest2', [structural_test_pattern], expected_result)
def frequencyPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LOCM", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=5))
runTest("nonFrequency", [pattern], createTestFile)
def structuralTest3():
"""
Seq([a, KC(b)])
"""
structural_test_pattern = Pattern(
SeqOperator(
PrimitiveEventStructure("GOOG", "a"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "b"))
),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=3)
)
expected_result = ('Seq', 'a', ('KC', 'b'))
runStructuralTest('structuralTest3', [structural_test_pattern], expected_result)
def structuralTest1():
"""
Seq([a, KC(And([KC(d), KC(Seq([e, f]))]))])
"""
structural_test_pattern = Pattern(
SeqOperator(PrimitiveEventStructure("GOOG", "a"),
KleeneClosureOperator(
AndOperator(PrimitiveEventStructure("GOOG", "b"),
KleeneClosureOperator(PrimitiveEventStructure("GOOG", "c"),
min_size=1, max_size=5),
KleeneClosureOperator(SeqOperator(PrimitiveEventStructure("GOOG", "d"), PrimitiveEventStructure("GOOG", "e")),
min_size=1, max_size=5)
),
min_size=1, max_size=5,
)),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
SimpleCondition(Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135)
),
timedelta(minutes=3)
)
expected_result = ('Seq', 'a', ('KC', ('And', ('And', 'b', ('KC', 'c')), ('KC', ('Seq', 'd', 'e')))))
runStructuralTest('structuralTest1', [structural_test_pattern], expected_result)
def arrivalRatesPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LOCM", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=5))
pattern.set_statistics(StatisticsTypes.ARRIVAL_RATES,
[0.0159, 0.0153, 0.0076])
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest("arrivalRates", [pattern], createTestFile, eval_params)
def nonFrequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=360))
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('nonFrequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def iiRandomPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y, z: x < y < z),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics({
StatisticsTypes.SELECTIVITY_MATRIX: selectivityMatrix,
StatisticsTypes.ARRIVAL_RATES: arrivalRates
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=IterativeImprovementTreePlanBuilderParameters(
DEFAULT_TREE_COST_MODEL, 20,
IterativeImprovementType.SWAP_BASED,
IterativeImprovementInitType.RANDOM),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[
StatisticsTypes.ARRIVAL_RATES,
StatisticsTypes.SELECTIVITY_MATRIX
])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('iiRandom1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def structuralTest6():
"""
Seq([a, Seq([ b, And([c, d]), e])])
"""
structural_test_pattern = Pattern(
SeqOperator(
PrimitiveEventStructure("GOOG", "a"),
SeqOperator(
PrimitiveEventStructure("GOOG", "b"),
AndOperator(
PrimitiveEventStructure("GOOG", "c"),
PrimitiveEventStructure("GOOG", "d")
),
PrimitiveEventStructure("GOOG", "e")
),
),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=3)
)
expected_result = ('Seq', 'a', ('Seq', ('Seq', 'b', ('And', 'c', 'd')), 'e'))
runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)
def structuralTest6():
"""
And([a, b, c, Seq([
d, KC(And([
e, KC(f), g
]),
And([ KC(h), KC(Seq([ i, j])
])
]),
k, l
])
"""
structural_test_pattern = Pattern(
AndOperator(
PrimitiveEventStructure("GOOG", "a"), PrimitiveEventStructure("GOOG", "b"), PrimitiveEventStructure("GOOG", "c"),
SeqOperator(
PrimitiveEventStructure("GOOG", "d"),
KleeneClosureOperator(
AndOperator(
PrimitiveEventStructure("GOOG", "e"), KleeneClosureOperator(PrimitiveEventStructure("GOOG", "f")), PrimitiveEventStructure("GOOG", "g")
)
), AndOperator(
KleeneClosureOperator(PrimitiveEventStructure("GOOG", "h")),
KleeneClosureOperator(
SeqOperator(
PrimitiveEventStructure("GOOG", "i"), PrimitiveEventStructure("GOOG", "j")
),
),
),
),
PrimitiveEventStructure("GOOG", "k"), PrimitiveEventStructure("GOOG", "l")
),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=3)
)
expected_result = ('And', ('And', ('And', ('And', ('And', 'a', 'b'), 'c'),
('Seq', ('Seq', 'd', ('KC', ('And', ('And', 'e', ('KC', 'f')), 'g'))),
('And', ('KC', 'h'), ('KC', ('Seq', 'i', 'j'))))), 'k'), 'l')
runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)
