def distinctPatterns(createTestFile=False):
pattern1 = Pattern(
SeqOperator(PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("AMZN", "x1"),
PrimitiveEventStructure("AMZN", "x2"),
PrimitiveEventStructure("AMZN", "x3")),
AndCondition(
SmallerThanEqCondition(Variable("x1", lambda x: x["Lowest Price"]),
75),
GreaterThanEqCondition(Variable("x2", lambda x: x["Peak Price"]),
78),
SmallerThanEqCondition(Variable("x3", lambda x: x["Lowest Price"]),
Variable("x1",
lambda x: x["Lowest Price"]))),
timedelta(days=1))
runMultiTest("BigMultiPattern", [pattern1, pattern2], createTestFile,
leaf_sharing_eval_mechanism_params)
def __init__(self, pattern_structure: PatternStructure, pattern_matching_condition: Condition,
time_window: timedelta, consumption_policy: ConsumptionPolicy = None, pattern_id: int = None):
self.id = pattern_id
self.full_structure = pattern_structure
self.positive_structure = pattern_structure.duplicate()
self.negative_structure = self.__extract_negative_structure()
self.condition = pattern_matching_condition
if isinstance(self.condition, TrueCondition):
self.condition = AndCondition()
elif not isinstance(self.condition, CompositeCondition):
self.condition = AndCondition(self.condition)
self.window = time_window
self.statistics_type = StatisticsTypes.NO_STATISTICS
self.statistics = None
self.consumption_policy = consumption_policy
if consumption_policy is not None:
if consumption_policy.single_event_strategy is not None and consumption_policy.single_types is None:
# must be initialized to contain all event types in the pattern
consumption_policy.single_types = self.get_all_event_types()
if consumption_policy.contiguous_names is not None:
self.__init_strict_conditions(pattern_structure)
def zStreamPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
AndCondition(
SmallerThanCondition(
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])))),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics(
StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES,
(selectivityMatrix, arrivalRates))
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(TreePlanBuilderTypes.ZSTREAM_BUSHY_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('zstream1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def samePatternDifferentTimeStampsFullSharing(createTestFile=False):
pattern1 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanEqCondition(Variable("a", lambda x: x["Peak Price"]),
135),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanEqCondition(Variable("a", lambda x: x["Peak Price"]),
135),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=2))
runMultiTest("DifferentTimeStampFullSharing", [pattern1, pattern2],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="DifferentTimeStamp")
def multipleParentsForInternalNode(createTestFile=False):
pattern1 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
500)), timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
530)), timedelta(minutes=3))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("FB", "e")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("e", lambda x: x["Peak Price"]),
520)), timedelta(minutes=5))
pattern4 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LI", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
100)), timedelta(minutes=2))
eval_mechanism_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL),
TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
MultiPatternEvaluationParameters(
MultiPatternEvaluationApproaches.SUBTREES_UNION))
runMultiTest("multipleParentsForInternalNode",
[pattern1, pattern2, pattern3, pattern4], createTestFile,
eval_mechanism_params)
def notInTheBeginningShare(createTestFile=False):
getattr_func = lambda x: x["Opening Price"]
pattern1 = Pattern(
SeqOperator(NegationOperator(PrimitiveEventStructure("TYP1", "x")),
NegationOperator(PrimitiveEventStructure("TYP2", "y")),
NegationOperator(PrimitiveEventStructure("TYP3", "z")),
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
SmallerThanCondition(Variable("b", getattr_func),
Variable("c", getattr_func))),
timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(NegationOperator(PrimitiveEventStructure("TYP1", "x")),
NegationOperator(PrimitiveEventStructure("TYP2", "y")),
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b")),
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
timedelta(minutes=5))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
GreaterThanCondition(Variable("c", getattr_func),
Variable("b", getattr_func))),
timedelta(minutes=5))
eval_mechanism_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL),
TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
MultiPatternEvaluationParameters(
MultiPatternEvaluationApproaches.SUBTREES_UNION))
runMultiTest("MultipleNotBeginningShare", [pattern1, pattern2, pattern3],
createTestFile, eval_mechanism_params)
def KC_Condition_Failure_03(createTestFile=False):
"""
KC(And([a, b, c]))
"""
try:
pattern = Pattern(
KleeneClosureOperator(
AndOperator(
PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b"),
PrimitiveEventStructure("GOOG", "c")
), min_size=1, max_size=3
),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]), Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]), Variable("c", lambda x: x["Peak Price"])),
KCIndexCondition(names={'a', 'b', 'c'}, getattr_func=lambda x: x["Peak Price"],
relation_op=lambda x, y: x < 1 + y,
offset=-1, first_index=2)
),
timedelta(minutes=3)
)
except Exception as e:
print("Test KC_Condition_Failure_03 Succeeded")
return
print("Test KC_Condition_Failure_03 Failed")
def iiRandom2PatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]),
relation_op=lambda x, y, z, w: x < y < z < w)),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics(
StatisticsTypes.SELECTIVITY_MATRIX_AND_ARRIVAL_RATES,
(selectivityMatrix, arrivalRates))
eval_params = TreeBasedEvaluationMechanismParameters(
IterativeImprovementTreePlanBuilderParameters(
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params.
cost_model_type, 20, IterativeImprovementType.CIRCLE_BASED,
IterativeImprovementInitType.RANDOM),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('iiRandom2', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def run_twitter_sanity_check():
"""
This basic test invokes a simple pattern looking for two tweets that retweeted the same tweet.
It might help finding users with common interests.
PATTERN SEQ(Tweet a, Tweet b)
WHERE a.Retweeted_Status_Id != None AND a.ID != b.ID AND a.Retweeted_Status_Id == b.Retweeted_Status_Id
"""
get_retweeted_status_function = lambda x: x[
"retweeted_status"] if "retweeted_status" in x else None
pattern_retweet = Pattern(
SeqOperator(
PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE,
"a"),
PrimitiveEventStructure(DummyTwitterEventTypeClassifier.TWEET_TYPE,
"b")),
AndCondition(
NotEqCondition(Variable("a", lambda x: x["id"]),
Variable("b", lambda x: x["id"])),
SimpleCondition(Variable("a", get_retweeted_status_function),
relation_op=lambda x: x is not None),
EqCondition(Variable("a", get_retweeted_status_function),
Variable("b", get_retweeted_status_function))),
timedelta(minutes=30))
cep = CEP([pattern_retweet])
event_stream = TwitterInputStream(['corona'])
try:
running_time = cep.run(
event_stream, FileOutputStream(os.getcwd(), "output.txt", True),
TweetDataFormatter())
print("Test twitterSanityCheck result: Succeeded, Time Passed: %s" %
(running_time, ))
finally:
event_stream.close()
def sortedStorageBenchMarkTest(createTestFile=False):
pattern = Pattern(
AndOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c"),
PrimitiveEventStructure("MSFT", "m")),
AndCondition(
GreaterThanEqCondition(Variable("b", lambda x: x["Lowest Price"]),
Variable("a", lambda x: x["Lowest Price"])),
GreaterThanEqCondition(Variable("m", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])),
GreaterThanEqCondition(Variable("m", lambda x: x["Lowest Price"]),
Variable("b", lambda x: x["Lowest Price"])),
),
timedelta(minutes=360),
)
runBenchMark("sortedStorageBenchMark - unsorted storage", [pattern])
storage_params = TreeStorageParameters(sort_storage=True,
attributes_priorities={
"a": 122,
"b": 200,
"c": 104,
"m": 139
})
eval_params = TreeBasedEvaluationMechanismParameters(
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.tree_plan_params,
storage_params)
runBenchMark("sortedStorageBenchMark - sorted storage", [pattern],
eval_mechanism_params=eval_params)
def msftDrivRacePatternSearchTest(createTestFile=False,
eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS,
test_name = "msftDrivRace"):
"""
This pattern is looking for a race between driv and microsoft in ten minutes
PATTERN SEQ(MicrosoftStockPriceUpdate a, DrivStockPriceUpdate b, MicrosoftStockPriceUpdate c, DrivStockPriceUpdate d, MicrosoftStockPriceUpdate e)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < d.PeakPrice AND d.PeakPrice < e.PeakPrice
WITHIN 10 minutes
"""
msftDrivRacePattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"), PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("MSFT", "c"), PrimitiveEventStructure("DRIV", "d"),
PrimitiveEventStructure("MSFT", "e")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("d", lambda x: x["Peak Price"]),
Variable("e", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y)
),
timedelta(minutes=10)
)
runTest(test_name, [msftDrivRacePattern], createTestFile, eval_mechanism_params)
def sortedStorageBenchMarkTest(createTestFile=False):
pattern = Pattern(
AndOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c"),
PrimitiveEventStructure("MSFT", "m")),
AndCondition(
GreaterThanEqCondition(Variable("b", lambda x: x["Lowest Price"]),
Variable("a", lambda x: x["Lowest Price"])),
GreaterThanEqCondition(Variable("m", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])),
GreaterThanEqCondition(Variable("m", lambda x: x["Lowest Price"]),
Variable("b", lambda x: x["Lowest Price"])),
),
timedelta(minutes=360),
)
runBenchMark("sortedStorageBenchMark - unsorted storage", [pattern])
storage_params = TreeStorageParameters(sort_storage=True,
attributes_priorities={
"a": 122,
"b": 200,
"c": 104,
"m": 139
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters()),
storage_params=storage_params)
runBenchMark("sortedStorageBenchMark - sorted storage", [pattern],
eval_mechanism_params=eval_params)
def severalPatternShareSubtreeFullSharing(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c"),
NegationOperator(PrimitiveEventStructure("TYP1", "x")),
NegationOperator(PrimitiveEventStructure("TYP2", "y")),
NegationOperator(PrimitiveEventStructure("TYP3", "z"))),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("TYP1", "x")),
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
timedelta(minutes=5))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b")),
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
timedelta(minutes=5))
runMultiTest("threeSharingSubtreesFullSharing",
[pattern, pattern2, pattern3],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="threeSharingSubtrees")
def frequencyPatternSearch2Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("LOCM", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
pattern.set_statistics(
{StatisticsTypes.ARRIVAL_RATES: [0.0076, 0.0153, 0.0159]})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest("frequency2", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
def nonsensePatternSearchTest(createTestFile=False,
eval_mechanism_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS,
test_name = "nonsense"):
"""
This pattern is looking for something that does not make sense.
PATTERN AND(AmazonStockPriceUpdate a, AvidStockPriceUpdate b, AppleStockPriceUpdate c)
WHERE a.PeakPrice < b.PeakPrice AND b.PeakPrice < c.PeakPrice AND c.PeakPrice < a.PeakPrice
"""
nonsensePattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"), PrimitiveEventStructure("AVID", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y),
BinaryCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("a", lambda x: x["Peak Price"]),
relation_op=lambda x, y: x < y)
),
timedelta(minutes=1)
)
runTest(test_name, [nonsensePattern], createTestFile, eval_mechanism_params)
def __init__(self, sliding_window: timedelta, parents, pattern_ids: int or Set[int] = None):
self._parents = []
self._sliding_window = sliding_window
self._partial_matches = None
self._condition = AndCondition()
# Full pattern matches that were not yet reported. Only relevant for an output node, that is, for a node
# corresponding to a full pattern definition.
self._unreported_matches = Queue()
self._is_output_node = False
# set of event types that will only appear in a single full match
self._single_event_types = set()
# events that were added to a partial match and cannot be added again
self._filtered_events = set()
# set of pattern IDs with which this node is associated
if pattern_ids is None:
pattern_ids = set()
elif isinstance(pattern_ids, int):
pattern_ids = {pattern_ids}
self._pattern_ids = pattern_ids
# Maps parent to event definition. This field helps to pass the parents a partial match with
# the right event definitions.
self._parent_to_info_dict = {}
# matches that were not yet pushed to the parents for further processing
self._parent_to_unhandled_queue_dict = {}
self.set_parents(parents, on_init=True)
def onePatternIncludesOther(createTestFile=False):
pattern1 = Pattern(
SeqOperator(PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("GOOG", "a"),
PrimitiveEventStructure("GOOG", "b")),
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
timedelta(minutes=3))
eval_mechanism_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL),
TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
MultiPatternEvaluationParameters(
MultiPatternEvaluationApproaches.SUBTREES_UNION))
runMultiTest("onePatternIncludesOther", [pattern1, pattern2],
createTestFile, eval_mechanism_params)
def frequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=360))
# frequencyDict = {"MSFT": 256, "DRIV": 257, "CBRL": 1}
pattern.set_statistics({
StatisticsTypes.ARRIVAL_RATES:
[0.01454418928322895, 0.016597077244258872, 0.012421711899791231]
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('frequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def samePatternSharingRoot(createTestFile=False):
hierarchyPattern = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"),
PrimitiveEventStructure("AAPL", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=1))
hierarchyPattern2 = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"),
PrimitiveEventStructure("AAPL", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=0.5))
hierarchyPattern3 = Pattern(
AndOperator(PrimitiveEventStructure("AMZN", "a"),
PrimitiveEventStructure("AAPL", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=0.1))
eval_mechanism_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL),
TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
MultiPatternEvaluationParameters(
MultiPatternEvaluationApproaches.SUBTREES_UNION))
runMultiTest('hierarchyMultiPattern',
[hierarchyPattern, hierarchyPattern2, hierarchyPattern3],
createTestFile, eval_mechanism_params)
def zstreamOrdNestedComplexStructuralTest():
pattern = Pattern(
AndOperator(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("DRIV", "c")),
SeqOperator(PrimitiveEventStructure("LOCM", "d"),
PrimitiveEventStructure("GOOG", "e")),
SeqOperator(
PrimitiveEventStructure("AVID", "f"),
PrimitiveEventStructure("BIDU", "g"),
SeqOperator(PrimitiveEventStructure("ORLY", "h"),
PrimitiveEventStructure("CBRL", "i"))),
PrimitiveEventStructure("MSFT", "j")),
AndCondition(
BinaryCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
BinaryCondition(Variable("d", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y: x > y),
EqCondition(Variable("a", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("b", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("c", lambda x: x["Date"]), 200802010900),
EqCondition(Variable("d", lambda x: x["Date"]), 200802010900)),
timedelta(minutes=3))
selectivityMatrix = [
[1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
[0.1, 1.0, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19],
[0.2, 0.12, 1.0, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29],
[0.3, 0.13, 0.23, 1.0, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39],
[0.4, 0.14, 0.24, 0.34, 1.0, 0.45, 0.46, 0.47, 0.48, 0.49],
[0.5, 0.15, 0.25, 0.35, 0.45, 1.0, 0.56, 0.57, 0.58, 0.59],
[0.6, 0.16, 0.26, 0.36, 0.46, 0.56, 1.0, 0.67, 0.68, 0.69],
[0.7, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 1.0, 0.78, 0.79],
[0.8, 0.18, 0.28, 0.38, 0.48, 0.58, 0.68, 0.78, 1.0, 0.89],
[0.9, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 1.0]
]
arrivalRates = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
pattern.set_statistics({
StatisticsTypes.ARRIVAL_RATES:
arrivalRates,
StatisticsTypes.SELECTIVITY_MATRIX:
selectivityMatrix
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=OptimizerParameters(
opt_type=OptimizerTypes.TRIVIAL_OPTIMIZER,
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.ORDERED_ZSTREAM_BUSHY_TREE)),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
expected_result = ('And', ('And', ('And', 'j', ('Seq', ('Seq', 'f', 'g'),
('Seq', 'h', 'i'))),
('Seq', 'd', 'e')), ('Seq', ('Seq', 'a', 'b'),
'c'))
runStructuralTest('zstreamOrdNestedComplexStructuralTest', [pattern],
expected_result,
eval_mechanism_params=eval_params)
def multipleParentsForInternalNodeFullSharing(createTestFile=False):
pattern1 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
500)), timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
530)), timedelta(minutes=3))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("FB", "e")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("e", lambda x: x["Peak Price"]),
520)), timedelta(minutes=5))
pattern4 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LI", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("c", lambda x: x["Peak Price"]),
100)), timedelta(minutes=2))
runMultiTest("multipleParentsForInternalNodeFullSharing",
[pattern1, pattern2, pattern3, pattern4],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="multipleParentsForInternalNode")
def __init__(self,
operator: OperatorTypes,
left_child: TreePlanNode,
right_child: TreePlanNode,
condition: CompositeCondition = AndCondition()):
super().__init__(operator, condition)
self.left_child = left_child
self.right_child = right_child
def __init__(self,
operator: OperatorTypes,
child: TreePlanNode,
index: int = 0,
condition: CompositeCondition = AndCondition()):
super().__init__(operator, condition)
self.child = child
self.index = index
def __init__(self,
operator: OperatorTypes,
positive_child: TreePlanNode,
negative_child: TreePlanNode,
is_unbounded: bool,
condition: CompositeCondition = AndCondition()):
super().__init__(operator, positive_child, negative_child, condition)
self.is_unbounded = is_unbounded
def __init__(self,
child: TreePlanNode,
index: int,
min_size: int,
max_size: int,
condition: CompositeCondition = AndCondition()):
super().__init__(OperatorTypes.KC, child, index, condition)
self.min_size = min_size
self.max_size = max_size
def __init__(self,
event_index: int,
event_type: str = None,
event_name: str = None,
condition: CompositeCondition = AndCondition()):
super().__init__(condition)
self.original_event_index = event_index # Keeps the node index in the nested subtree its in.
self.event_index = event_index # The index of the node in the main tree.
self.event_type = event_type
self.event_name = event_name
def KC_Specific_Value(createTestFile=False):
pattern = Pattern(
SeqOperator(KleeneClosureOperator(PrimitiveEventStructure("GOOG", "a"))),
AndCondition(
SimpleCondition(Variable("a", lambda x: x["Opening Price"]), relation_op=lambda x: x > 0),
KCValueCondition(names={'a'}, getattr_func=lambda x: x["Peak Price"], relation_op=lambda x, y: x > y, index=2, value=530.5)
),
timedelta(minutes=5)
)
runTest("KC_Specific_Value_", [pattern], createTestFile, events=nasdaqEventStreamKC)
def notInTheBeginningShareFullSharing(createTestFile=False):
getattr_func = lambda x: x["Opening Price"]
pattern1 = Pattern(
SeqOperator(NegationOperator(PrimitiveEventStructure("TYP1", "x")),
NegationOperator(PrimitiveEventStructure("TYP2", "y")),
NegationOperator(PrimitiveEventStructure("TYP3", "z")),
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
SmallerThanCondition(Variable("b", getattr_func),
Variable("c", getattr_func))),
timedelta(minutes=5))
pattern2 = Pattern(
SeqOperator(NegationOperator(PrimitiveEventStructure("TYP1", "x")),
NegationOperator(PrimitiveEventStructure("TYP2", "y")),
PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b")),
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
timedelta(minutes=5))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", getattr_func),
Variable("b", getattr_func)),
GreaterThanCondition(Variable("c", getattr_func),
Variable("b", getattr_func))),
timedelta(minutes=5))
runMultiTest("MultipleNotBeginningShareFullSharing",
[pattern1, pattern2, pattern3],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="MultipleNotBeginningShare")
def threePatternsTestFullSharing(createTestFile=False):
pattern1 = Pattern(
AndOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"]))),
timedelta(minutes=1))
pattern2 = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("MSFT", "c"),
PrimitiveEventStructure("DRIV", "d"),
PrimitiveEventStructure("MSFT", "e")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"])),
SmallerThanCondition(Variable("d", lambda x: x["Peak Price"]),
Variable("e", lambda x: x["Peak Price"]))),
timedelta(minutes=10))
pattern3 = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("GOOG", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
runMultiTest("ThreePatternTestFullSharing", [pattern1, pattern2, pattern3],
createTestFile,
subtree_sharing_eval_mechanism_params,
expected_file_name="ThreePatternTest")
def nonFrequencyPatternSearch2Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("LOCM", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
runTest("nonFrequency2", [pattern], createTestFile)
