def frequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=360))
# frequencyDict = {"MSFT": 256, "DRIV": 257, "CBRL": 1}
pattern.set_statistics({
StatisticsTypes.ARRIVAL_RATES:
[0.01454418928322895, 0.016597077244258872, 0.012421711899791231]
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('frequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def frequencyPatternSearch2Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("LOCM", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
pattern.set_statistics(
{StatisticsTypes.ARRIVAL_RATES: [0.0076, 0.0153, 0.0159]})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest("frequency2", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
def frequencyPatternSearch6Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a1"),
PrimitiveEventStructure("LOCM", "b1"),
PrimitiveEventStructure("AAPL", "a2"),
PrimitiveEventStructure("LOCM", "b2"),
PrimitiveEventStructure("AAPL", "a3"),
PrimitiveEventStructure("LOCM", "b3")), TrueCondition(),
timedelta(minutes=7))
pattern.set_statistics({
StatisticsTypes.ARRIVAL_RATES:
[0.0159, 0.0076, 0.0159, 0.0076, 0.0159, 0.0076]
}) # {"AAPL": 460, "LOCM": 219}
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest("frequency6", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
def __init__(
self,
storage_params: TreeStorageParameters = TreeStorageParameters(),
optimizer_params:
OptimizerParameters = StatisticsDeviationAwareOptimizerParameters(),
tree_update_type: TreeEvaluationMechanismUpdateTypes = DefaultConfig.
DEFAULT_TREE_UPDATE_TYPE):
super().__init__(EvaluationMechanismTypes.TREE_BASED, optimizer_params)
self.storage_params = storage_params
self.tree_update_type = tree_update_type
def zStreamPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
AndCondition(
SmallerThanCondition(
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])))),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics({
StatisticsTypes.SELECTIVITY_MATRIX: selectivityMatrix,
StatisticsTypes.ARRIVAL_RATES: arrivalRates
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.ZSTREAM_BUSHY_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[
StatisticsTypes.ARRIVAL_RATES,
StatisticsTypes.SELECTIVITY_MATRIX
])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('zstream1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def nonFrequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=360))
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE)),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('nonFrequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
nasdaqEventStream_AAPL_AMZN_GOOG = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_AAPL_AMZN_GOOG.txt"))
nasdaqEventStream = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_LONG.txt"))
nasdaqEventStreamHalfShort = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_HALF_SHORT.txt"))
custom = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom.txt"))
custom2 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom2.txt"))
custom3 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom3.txt"))
nasdaqEventStreamKC = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_KC.txt"))
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS = \
TreeBasedEvaluationMechanismParameters(storage_params=TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
optimizer_params=StatisticsDeviationAwareOptimizerParameters(tree_plan_params
=TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL)))
"""
Default testing Data formatters
"""
DEFAULT_TESTING_DATA_FORMATTER = MetastockDataFormatter()
class FailedCounter:
"""
This class helps tracking failed tests (if there are any).
"""
counter = 0
failed_tests = set()
missing_combination = []
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL)
DEFAULT_TESTING_INVARIANT_AWARE_ZSTREAM_BUSHY_TREE_BUILDER = \
TreePlanBuilderParameters(TreePlanBuilderTypes.INVARIANT_AWARE_ZSTREAM_BUSHY_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL)
"""
Default testing optimizer settings
"""
DEFAULT_TESTING_TRIVIAL_OPTIMIZER_SETTINGS = \
TrivialOptimizerParameters(tree_plan_params=DEFAULT_BASIC_TESTING_TREE_BUILDER,
statistics_collector_params=DEFAULT_TESTING_STATISTICS_COLLECTOR_SELECTIVITY_AND_ARRIVAL_RATES_STATISTICS,
statistics_updates_wait_time=timedelta(minutes=10))
DEFAULT_TESTING_DEVIATION_AWARE_OPTIMIZER_SETTINGS = \
StatisticsDeviationAwareOptimizerParameters(tree_plan_params=DEFAULT_BASIC_TESTING_TREE_BUILDER, deviation_threshold=0.5,
statistics_collector_params=DEFAULT_TESTING_STATISTICS_COLLECTOR_SELECTIVITY_AND_ARRIVAL_RATES_STATISTICS,
statistics_updates_wait_time=timedelta(minutes=10))
DEFAULT_TESTING_GREEDY_INVARIANT_OPTIMIZER_SETTINGS = \
InvariantsAwareOptimizerParameters(tree_plan_params=DEFAULT_TESTING_INVARIANT_AWARE_GREEDY_TREE_BUILDER,
statistics_collector_params=DEFAULT_TESTING_STATISTICS_COLLECTOR_SELECTIVITY_AND_ARRIVAL_RATES_STATISTICS,
statistics_updates_wait_time=timedelta(minutes=10))
DEFAULT_TESTING_ZSTREAM_INVARIANT_OPTIMIZER_SETTINGS = \
InvariantsAwareOptimizerParameters(tree_plan_params=DEFAULT_TESTING_INVARIANT_AWARE_ZSTREAM_BUSHY_TREE_BUILDER,
statistics_collector_params=DEFAULT_TESTING_STATISTICS_COLLECTOR_SELECTIVITY_AND_ARRIVAL_RATES_STATISTICS,
statistics_updates_wait_time=timedelta(minutes=10))
"""
Default testing Evaluation mechanism settings
"""
"""