def greedyNestedStructuralTest():
pattern = Pattern(
AndOperator(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b")),
SeqOperator(PrimitiveEventStructure("AVID", "c"),
PrimitiveEventStructure("BIDU", "d"))),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
BinaryCondition(Variable("d", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
EqCondition(Variable("a", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("b", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("c", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("d", lambda x: x["Date"]), 200802010900)),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics({
StatisticsTypes.ARRIVAL_RATES:
arrivalRates,
StatisticsTypes.SELECTIVITY_MATRIX:
selectivityMatrix
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=OptimizerParameters(
opt_type=OptimizerTypes.TRIVIAL_OPTIMIZER,
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.GREEDY_LEFT_DEEP_TREE)),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
expected_result = ('And', ('Seq', 'b', 'a'), ('Seq', 'd', 'c'))
runStructuralTest('greedyNestedStructuralTest', [pattern],
expected_result,
eval_mechanism_params=eval_params)
def KC_AND(createTestFile=False):
"""
KC(And([a, b, c]))
"""
pattern = Pattern(
KleeneClosureOperator(
AndOperator(
PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b"),
PrimitiveEventStructure("GOOG", "c")
), min_size=1, max_size=3
),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"])),
),
timedelta(minutes=3)
)
runTest("KC_AND_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def KC_AND_IndexCondition_02(createTestFile=False):
"""
KC(And([a, b]))
"""
pattern = Pattern(
KleeneClosureOperator(
AndOperator(
PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b")
), min_size=1, max_size=3
),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])),
KCIndexCondition(names={'a', 'b'}, getattr_func=lambda x: x["Peak Price"], relation_op=lambda x, y: x < y,
offset=2),
),
timedelta(minutes=3)
)
runTest("KC_AND_IndexCondition_02_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def threePatternsTestFullSharing(createTestFile=False):
pattern1 = Pattern(
AndOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=1))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("MSFT", "c"),
PrimitiveEventStructure("DRIV", "d"),
PrimitiveEventStructure("MSFT", "e")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("d", lambda x: x["Peak Price"]),
Variable("e", lambda x: x["Peak Price"]))),
timedelta(minutes=10))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
runMultiTest("ThreePatternTestFullSharing", [pattern1, pattern2, pattern3],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="ThreePatternTest")
def hierarchyPatternSearchTest(createTestFile=False):
"""
The following pattern is looking for Amazon < Apple < Google cases in one minute windows.
PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice
WITHIN 1 minute
"""
hierarchyPattern = Pattern(
AndOperator(
[QItem("AMZN", "a"),
QItem("AAPL", "b"),
QItem("GOOG", "c")]),
AndFormula(
SmallerThanFormula(IdentifierTerm("a", lambda x: x["Peak Price"]),
IdentifierTerm("b", lambda x: x["Peak Price"])),
SmallerThanFormula(IdentifierTerm("b", lambda x: x["Peak Price"]),
IdentifierTerm("c",
lambda x: x["Peak Price"]))),
timedelta(minutes=1))
runTest('hierarchy', [hierarchyPattern], createTestFile)
def hierarchyPatternSearchTest(createTestFile=False):
"""
The following pattern is looking for Amazon < Apple < Google cases in one minute windows.
PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice
WITHIN 1 minute
"""
hierarchyPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AAPL", "b"), PrimitiveEventStructure("GOOG", "c")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y)
),
timedelta(minutes=1)
)
runTest('hierarchy', [hierarchyPattern], createTestFile)
def googleAmazonLowPatternSearchTest(createTestFile=False,
eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS,
test_name = "googleAmazonLow"):
"""
This pattern is looking for low prices of Amazon and Google at the same minute.
PATTERN AND(AmazonStockPriceUpdate a, GoogleStockPriceUpdate g)
WHERE a.PeakPrice <= 73 AND g.PeakPrice <= 525
WITHIN 1 minute
"""
googleAmazonLowPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("GOOG", "g")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 73),
SimpleCondition(Variable("g", lambda x: x["Peak Price"]),
relation_op=lambda x: x <= 525)
),
timedelta(minutes=1)
)
runTest(test_name, [googleAmazonLowPattern], createTestFile, eval_mechanism_params)
def structuralTest6():
"""
Seq([a, Seq([ b, And([c, d]), e])])
"""
structural_test_pattern = Pattern(
SeqOperator(
PrimitiveEventStructure("GOOG", "a"),
SeqOperator(
PrimitiveEventStructure("GOOG", "b"),
AndOperator(
PrimitiveEventStructure("GOOG", "c"),
PrimitiveEventStructure("GOOG", "d")
),
PrimitiveEventStructure("GOOG", "e")
),
),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
timedelta(minutes=3)
)
expected_result = ('Seq', 'a', ('Seq', ('Seq', 'b', ('And', 'c', 'd')), 'e'))
runStructuralTest('structuralTest6', [structural_test_pattern], expected_result)
def sortedStorageTest(createTestFile=False):
pattern = Pattern(
AndOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"])),
),
timedelta(minutes=360),
)
storage_params = TreeStorageParameters(True, clean_up_interval=500)
eval_params = TreeBasedEvaluationMechanismParameters(
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params,
storage_params)
runTest("sortedStorageTest", [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def basicNestedTest(createTestFile=False):
pattern = Pattern(
AndOperator(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b")),
SeqOperator(PrimitiveEventStructure("AVID", "c"),
PrimitiveEventStructure("BIDU", "d"))),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
BinaryCondition(Variable("d", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
EqCondition(Variable("a", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("b", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("c", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("d", lambda x: x["Date"]), 200802010900)),
timedelta(minutes=5))
expected_result = ('And', ('Seq', 'a', 'b'), ('Seq', 'c', 'd'))
runStructuralTest('basicNestedStructuralTest', [pattern], expected_result)
runTest("basicNested", [pattern], createTestFile)
def nonsensePatternSearchTest(createTestFile=False):
"""
This pattern is looking for something that does not make sense.
PATTERN AND(AmazonStockPriceUpdate a, AvidStockPriceUpdate b, AppleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < a.PeakPrice
"""
nonsensePattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AVID", "b"), PrimitiveEventStructure("AAPL", "c")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("a", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y)
),
timedelta(minutes=1)
)
runTest('nonsense', [nonsensePattern], createTestFile)
def sortedStorageTest(createTestFile=False):
pattern = Pattern(
AndOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"])),
),
timedelta(minutes=360),
)
storage_params = TreeStorageParameters(True, clean_up_interval=500)
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters()),
storage_params=storage_params)
runTest("sortedStorageTest", [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def sortedStorageTest(createTestFile=False):
pattern = Pattern(
AndOperator(
[QItem("DRIV", "a"),
QItem("MSFT", "b"),
QItem("CBRL", "c")]),
AndFormula(
GreaterThanFormula(
IdentifierTerm("a", lambda x: x["Opening Price"]),
IdentifierTerm("b", lambda x: x["Opening Price"])),
GreaterThanFormula(
IdentifierTerm("b", lambda x: x["Opening Price"]),
IdentifierTerm("c", lambda x: x["Opening Price"])),
),
timedelta.max,
)
runTest(
"sortedStorageTest",
[pattern],
createTestFile,
eval_mechanism_type=EvaluationMechanismTypes.TRIVIAL_LEFT_DEEP_TREE,
events=nasdaqEventStream,
)
def nonsensePatternSearchTest(createTestFile=False):
"""
This pattern is looking for something that does not make sense.
PATTERN AND(AmazonStockPriceUpdate a, AvidStockPriceUpdate b, AppleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < a.PeakPrice
"""
nonsensePattern = Pattern(
AndOperator(
[QItem("AMZN", "a"),
QItem("AVID", "b"),
QItem("AAPL", "c")]),
AndFormula(
SmallerThanFormula(IdentifierTerm("a", lambda x: x["Peak Price"]),
IdentifierTerm("b", lambda x: x["Peak Price"])),
AndFormula(
SmallerThanFormula(
IdentifierTerm("b", lambda x: x["Peak Price"]),
IdentifierTerm("c", lambda x: x["Peak Price"])),
SmallerThanFormula(
IdentifierTerm("c", lambda x: x["Peak Price"]),
IdentifierTerm("a", lambda x: x["Peak Price"])))),
timedelta(minutes=1))
runTest('nonsense', [nonsensePattern], createTestFile)
def hierarchyPatternSearchTest(createTestFile=False,
eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS,
test_name = "hierarchy"):
"""
The following pattern is looking for Amazon < Apple < Google cases in one minute windows.
PATTERN AND(AmazonStockPriceUpdate a, AppleStockPriceUpdate b, GoogleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice
WITHIN 1 minute
"""
hierarchyPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AAPL", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y)
),
timedelta(minutes=1)
)
runTest(test_name, [hierarchyPattern], createTestFile, eval_mechanism_params)
def structuralTest1():
"""
Seq([a, KC(And([KC(d), KC(Seq([e, f]))]))])
"""
structural_test_pattern = Pattern(
SeqOperator(PrimitiveEventStructure("GOOG", "a"),
KleeneClosureOperator(
AndOperator(PrimitiveEventStructure("GOOG", "b"),
KleeneClosureOperator(PrimitiveEventStructure("GOOG", "c"),
min_size=1, max_size=5),
KleeneClosureOperator(SeqOperator(PrimitiveEventStructure("GOOG", "d"), PrimitiveEventStructure("GOOG", "e")),
min_size=1, max_size=5)
),
min_size=1, max_size=5,
)),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135),
SimpleCondition(Variable("b", lambda x: x["Opening Price"]), relation_op=lambda x: x > 135)
),
timedelta(minutes=3)
)
expected_result = ('Seq', 'a', ('KC', ('And', ('And', 'b', ('KC', 'c')), ('KC', ('Seq', 'd', 'e')))))
runStructuralTest('structuralTest1', [structural_test_pattern], expected_result)
def threePatternsTest(createTestFile=False):
pattern1 = Pattern(
AndOperator(
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
),
AndCondition(
SmallerThanCondition(
Variable("a", lambda x: x["Peak Price"] - x["Opening Price"]),
Variable("b", lambda x: x["Peak Price"] - x["Opening Price"]),
), ),
timedelta(minutes=1),
confidence=0.96)
pattern2 = Pattern(
SeqOperator(
PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("MSFT", "c"),
PrimitiveEventStructure("DRIV", "d"),
PrimitiveEventStructure("MSFT", "e"),
),
AndCondition(
SmallerThanCondition(
Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
),
SmallerThanCondition(
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
),
SmallerThanCondition(
Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]),
),
SmallerThanCondition(
Variable("d", lambda x: x["Peak Price"]),
Variable("e", lambda x: x["Peak Price"]),
),
),
timedelta(minutes=10),
confidence=0.6,
)
pattern3 = Pattern(
SeqOperator(
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c"),
),
AndCondition(
GreaterThanCondition(
Variable("a", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
),
GreaterThanCondition(
Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
),
),
timedelta(minutes=5),
confidence=0.6,
)
runMultiTest("ThreePatternTestProb", [pattern1, pattern2, pattern3],
createTestFile,
eventStream=nasdaqEventStreamP)