def arrivalRatesPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("LOCM", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=5))
pattern.set_statistics(StatisticsTypes.ARRIVAL_RATES,
[0.0159, 0.0153, 0.0076])
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest("arrivalRates", [pattern], createTestFile, eval_params)
def frequencyPatternSearch3Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AAPL", "b"),
PrimitiveEventStructure("AAPL", "c"),
PrimitiveEventStructure("LOCM", "d")), TrueCondition(),
timedelta(minutes=5))
pattern.set_statistics(StatisticsTypes.FREQUENCY_DICT, {
"AAPL": 460,
"LOCM": 219
})
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest("frequency3", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
def zStreamPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("MSFT", "a"),
PrimitiveEventStructure("DRIV", "b"),
PrimitiveEventStructure("ORLY", "c"),
PrimitiveEventStructure("CBRL", "d")),
AndCondition(
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Peak Price"]),
Variable("b", lambda x: x["Peak Price"])),
AndCondition(
SmallerThanCondition(
Variable("b", lambda x: x["Peak Price"]),
Variable("c", lambda x: x["Peak Price"])))),
SmallerThanCondition(Variable("c", lambda x: x["Peak Price"]),
Variable("d", lambda x: x["Peak Price"]))),
timedelta(minutes=3))
selectivityMatrix = [[1.0, 0.9457796098355941, 1.0, 1.0],
[0.9457796098355941, 1.0, 0.15989723367389616, 1.0],
[1.0, 0.15989723367389616, 1.0, 0.9992557393942864],
[1.0, 1.0, 0.9992557393942864, 1.0]]
arrivalRates = [
0.016597077244258872, 0.01454418928322895, 0.013917884481558803,
0.012421711899791231
]
pattern.set_statistics({
StatisticsTypes.SELECTIVITY_MATRIX: selectivityMatrix,
StatisticsTypes.ARRIVAL_RATES: arrivalRates
})
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.ZSTREAM_BUSHY_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[
StatisticsTypes.ARRIVAL_RATES,
StatisticsTypes.SELECTIVITY_MATRIX
])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('zstream1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def nonFrequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
SimpleCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]),
relation_op=lambda x, y, z: x > y > z),
timedelta(minutes=360))
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE)),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
runTest('nonFrequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def frequencyPatternSearch4Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("AAPL", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("AVID", "c"),
PrimitiveEventStructure("LOCM", "d")), TrueCondition(),
timedelta(minutes=7))
eval_params = TreeBasedEvaluationMechanismParameters(
optimizer_params=StatisticsDeviationAwareOptimizerParameters(
tree_plan_params=TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
statistics_collector_params=StatisticsCollectorParameters(
statistics_types=[StatisticsTypes.ARRIVAL_RATES])),
storage_params=DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.
storage_params)
pattern.set_statistics(
{StatisticsTypes.ARRIVAL_RATES: [0.0159, 0.0153, 0.0146, 0.0076]})
runTest("frequency4", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
def frequencyTailoredPatternSearchTest(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("DRIV", "a"),
PrimitiveEventStructure("MSFT", "b"),
PrimitiveEventStructure("CBRL", "c")),
AndCondition(
GreaterThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
GreaterThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=360))
frequencyDict = {"MSFT": 256, "DRIV": 257, "CBRL": 1}
pattern.set_statistics(StatisticsTypes.FREQUENCY_DICT, frequencyDict)
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest('frequencyTailored1', [pattern],
createTestFile,
eval_mechanism_params=eval_params,
events=nasdaqEventStream)
def frequencyPatternSearch2Test(createTestFile=False):
pattern = Pattern(
SeqOperator(PrimitiveEventStructure("LOCM", "a"),
PrimitiveEventStructure("AMZN", "b"),
PrimitiveEventStructure("AAPL", "c")),
AndCondition(
SmallerThanCondition(Variable("a", lambda x: x["Opening Price"]),
Variable("b", lambda x: x["Opening Price"])),
SmallerThanCondition(Variable("b", lambda x: x["Opening Price"]),
Variable("c", lambda x: x["Opening Price"]))),
timedelta(minutes=5))
pattern.set_statistics(StatisticsTypes.FREQUENCY_DICT, {
"AAPL": 2,
"AMZN": 3,
"LOCM": 1
})
eval_params = TreeBasedEvaluationMechanismParameters(
TreePlanBuilderParameters(
TreePlanBuilderTypes.SORT_BY_FREQUENCY_LEFT_DEEP_TREE),
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS.storage_params)
runTest("frequency2", [pattern],
createTestFile,
eval_mechanism_params=eval_params)
nasdaqEventStreamFrequencyTailored = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_FREQUENCY_TAILORED.txt"))
nasdaqEventStream_AAPL_AMZN_GOOG = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_AAPL_AMZN_GOOG.txt"))
nasdaqEventStream = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_LONG.txt"))
nasdaqEventStreamHalfShort = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_HALF_SHORT.txt"))
custom = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom.txt"))
custom2 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom2.txt"))
custom3 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom3.txt"))
nasdaqEventStreamKC = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_KC.txt"))
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS = \
TreeBasedEvaluationMechanismParameters(storage_params=TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True),
optimizer_params=StatisticsDeviationAwareOptimizerParameters(tree_plan_params
=TreePlanBuilderParameters(
TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL)))
"""
Default testing Data formatters
"""
DEFAULT_TESTING_DATA_FORMATTER = MetastockDataFormatter()
class FailedCounter:
"""
This class helps tracking failed tests (if there are any).
"""
counter = 0
failed_tests = set()
statistics_updates_wait_time=timedelta(minutes=10))
DEFAULT_TESTING_ZSTREAM_INVARIANT_OPTIMIZER_SETTINGS = \
InvariantsAwareOptimizerParameters(tree_plan_params=DEFAULT_TESTING_INVARIANT_AWARE_ZSTREAM_BUSHY_TREE_BUILDER,
statistics_collector_params=DEFAULT_TESTING_STATISTICS_COLLECTOR_SELECTIVITY_AND_ARRIVAL_RATES_STATISTICS,
statistics_updates_wait_time=timedelta(minutes=10))
"""
Default testing Evaluation mechanism settings
"""
"""
evaluation mechanism: trivial
optimizer: trivial
"""
DEFAULT_TESTING_TRIVIAL_EVALUATION_MECHANISM_SETTINGS = \
TreeBasedEvaluationMechanismParameters(storage_params=DEFAULT_TREE_STORAGE_PARAMETERS,
tree_update_type=TreeEvaluationMechanismUpdateTypes.TRIVIAL_TREE_EVALUATION,
optimizer_params=DEFAULT_TESTING_TRIVIAL_OPTIMIZER_SETTINGS)
"""
evaluation mechanism: trivial
optimizer: changes aware optimizer
"""
DEFAULT_TESTING_TRIVIAL_EVALUATION_MECHANISM_SETTINGS_AND_DEVIATION_AWARE_OPTIMIZER = \
TreeBasedEvaluationMechanismParameters(storage_params=DEFAULT_TREE_STORAGE_PARAMETERS,
tree_update_type=TreeEvaluationMechanismUpdateTypes.TRIVIAL_TREE_EVALUATION,
optimizer_params=DEFAULT_TESTING_DEVIATION_AWARE_OPTIMIZER_SETTINGS)
"""
evaluation mechanism: trivial
optimizer: greedy invariant
"""
DEFAULT_TESTING_TRIVIAL_EVALUATION_MECHANISM_SETTINGS_AND_GREEDY_INVARIANT_OPTIMIZER = \
TreeBasedEvaluationMechanismParameters(storage_params=DEFAULT_TREE_STORAGE_PARAMETERS,
nasdaqEventStreamMedium = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_MEDIUM.txt"))
nasdaqEventStreamFrequencyTailored = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_FREQUENCY_TAILORED.txt"))
nasdaqEventStream_AAPL_AMZN_GOOG = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_AAPL_AMZN_GOOG.txt"))
nasdaqEventStream = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_LONG.txt"))
nasdaqEventStreamHalfShort = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_HALF_SHORT.txt"))
custom = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom.txt"))
custom2 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom2.txt"))
custom3 = FileInputStream(os.path.join(absolutePath, "test/EventFiles/custom3.txt"))
nasdaqEventStreamKC = FileInputStream(os.path.join(absolutePath, "test/EventFiles/NASDAQ_KC.txt"))
DEFAULT_TESTING_EVALUATION_MECHANISM_SETTINGS = \
TreeBasedEvaluationMechanismParameters(TreePlanBuilderParameters(TreePlanBuilderTypes.TRIVIAL_LEFT_DEEP_TREE,
TreeCostModels.INTERMEDIATE_RESULTS_TREE_COST_MODEL),
TreeStorageParameters(sort_storage=False,
clean_up_interval=10,
prioritize_sorting_by_timestamp=True))
DEFAULT_TESTING_DATA_FORMATTER = MetastockDataFormatter()
def numOfLinesInPattern(file):
"""
get num of lines in file until first blank line == num of lines in pattern
:param file: file
:return: int
"""
counter = 0
for line in file:
if line == '\n':
break
counter = counter + 1
