def test_expression_valid_single_value() -> None:
code_string = 1
expr = Expression(code_string)
assert expr.code_string == str(code_string)
assert expr.evaluate(EvaluationContext()) == 1
code_string = False
expr = Expression(code_string)
assert expr.code_string == str(code_string)
assert expr.evaluate(EvaluationContext()) is False
code_string = '"Hello World"'
expr = Expression(code_string)
assert expr.code_string == code_string
assert expr.evaluate(EvaluationContext()) == "Hello World"
code_string = '[1, 2, 3]'
expr = Expression(code_string)
assert expr.code_string == code_string
assert expr.evaluate(EvaluationContext()) == [1, 2, 3]
code_string = '{"a":1, "b":2}'
expr = Expression(code_string)
assert expr.code_string == code_string
assert expr.evaluate(EvaluationContext()) == {"a": 1, "b": 2}
def test_anchor_max_two(anchor_schema_max_two: AnchorSchema,
block_item: BlockAggregate) -> None:
anchor = Anchor(anchor_schema_max_two)
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is True
anchor.add_condition_met()
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is True
anchor.add_condition_met()
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is False
def test_anchor_max_not_specified(anchor_schema_max_one: AnchorSchema,
block_item: BlockAggregate) -> None:
anchor_schema_max_one.max = None
anchor = Anchor(anchor_schema_max_one)
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is True
anchor.add_condition_met()
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is True
anchor.add_condition_met()
assert anchor.evaluate_anchor(block_item, EvaluationContext()) is True
def context(self) -> EvaluationContext:
if self._context:
return self._context
# The eval code adds the python global context to the global context dict being passed and
# new context being created is added to the local context. We take the local_context in
# temp_eval_context and use that as the global context for the returned EvaluationContext.
temp_eval_context = EvaluationContext()
for import_statement in self.import_statements:
import_statement.evaluate(temp_eval_context)
eval_context = EvaluationContext(
global_context=temp_eval_context.local_context)
return eval_context
def test_expression_globals_locals() -> None:
code_string = 'a + b + 1'
expr = Expression(code_string)
with pytest.raises(NameError, match='name \'a\' is not defined'):
expr.evaluate(EvaluationContext())
assert expr.evaluate(EvaluationContext(Context({'a': 2, 'b': 3}))) == 6
assert expr.evaluate(
EvaluationContext(local_context=Context({
'a': 2,
'b': 3
}))) == 6
assert expr.evaluate(
EvaluationContext(Context({'a': 2}), Context({'b': 3}))) == 6
def create_block_aggregate(schema, time, identity) -> BlockAggregate:
evaluation_context = EvaluationContext()
block_aggregate = BlockAggregate(schema=schema,
identity=identity,
evaluation_context=evaluation_context)
evaluation_context.global_add('time', time)
evaluation_context.global_add('user', block_aggregate)
evaluation_context.global_add('identity', identity)
return block_aggregate
def test_execution_key_error(caplog) -> None:
caplog.set_level(logging.DEBUG)
code_string = 'test_dict[\'missing_key\'] + 1'
assert Expression(code_string).evaluate(
EvaluationContext(Context({'test_dict': {}}))) is None
assert 'KeyError in evaluating expression test_dict[\'missing_key\'] + 1. Error: \'missing_key\'' in caplog.records[
0].message
assert caplog.records[0].levelno == logging.DEBUG
def test_execution_error_type_mismatch(caplog) -> None:
caplog.set_level(logging.DEBUG)
code_string = '1 + \'a\''
assert Expression(code_string).evaluate(
EvaluationContext(Context({'test_dict': {}}))) is None
assert 'TypeError in evaluating expression 1 + \'a\'' in caplog.records[
0].message
assert caplog.records[0].levelno == logging.DEBUG
def test_aggregate_finalize(aggregate_schema_with_store):
aggregate = MockAggregate(schema=aggregate_schema_with_store,
identity="12345",
evaluation_context=EvaluationContext())
aggregate.run_finalize()
snapshot_aggregate = aggregate._store.get(
Key(identity="12345", group="user"))
assert snapshot_aggregate is not None
assert snapshot_aggregate == aggregate._snapshot
def test_expression_user_function() -> None:
code_string = '2 if test_function() else 3'
def test_function():
return 3 > 4
expr = Expression(code_string)
assert expr.evaluate(
EvaluationContext(Context({'test_function': test_function}))) == 3
def test_aggregate_persist_with_store(aggregate_schema_with_store):
aggregate = MockAggregate(
schema=aggregate_schema_with_store,
identity="12345",
evaluation_context=EvaluationContext())
aggregate._persist()
snapshot_aggregate = aggregate._store.get(
Key(KeyType.DIMENSION, identity="12345", group="user"))
assert snapshot_aggregate is not None
assert snapshot_aggregate == aggregate._snapshot
def test_get_attribute_invalid(collection_schema_spec: Dict[str, Any],
mock_nested_items: contextmanager) -> None:
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec(collection_schema_spec)
schema_collection = MockBaseSchemaCollection(
name, schema_loader, AggregateSchema.ATTRIBUTE_FIELDS)
with raises(Exception):
item_collection = MockBaseItemCollection(schema_collection,
EvaluationContext())
assert item_collection.event_counts
# Test nested items not specified
with mock_nested_items:
with raises(Exception):
item_collection = MockBaseItemCollection(schema_collection,
EvaluationContext())
mock_nested_items.return_value = None
assert item_collection.event_count
def test_aggregate_nested_items(aggregate_schema_with_store):
aggregate = MockAggregate(schema=aggregate_schema_with_store,
identity="12345",
evaluation_context=EvaluationContext())
nested_items = aggregate._nested_items
assert len(nested_items) == 2
assert "event_count" in nested_items
assert isinstance(nested_items["event_count"], Field)
assert "_identity" in nested_items
assert isinstance(nested_items["_identity"], Field)
def _load_blocks(self, blocks: List[Tuple[Key, Any]]) -> List[BlockAggregate]:
"""
Converts [(Key, block)] to [BlockAggregate]
:param blocks: List of (Key, block) blocks.
:return: List of BlockAggregate
"""
return [
BlockAggregate(self._schema.source, self._identity,
EvaluationContext()).run_restore(block) for (_, block) in blocks
]
def test_execution_error_missing_field(caplog,
schema_loader: SchemaLoader) -> None:
caplog.set_level(logging.DEBUG)
context = Context({
'test':
StreamingTransformer(schema_loader.get_schema_object('test'), 'user1')
})
with raises(MissingAttributeError,
match='missing_field not defined in test_group'):
Expression('test.test_group.missing_field').evaluate(
EvaluationContext(context))
assert (
'MissingAttributeError in evaluating expression test.test_group.missing_field. '
'Error: missing_field not defined in test_group') in caplog.text
with raises(MissingAttributeError,
match='missing_field not defined in test_group'):
Expression('test.test_group[\'missing_field\']').evaluate(
EvaluationContext(context))
def test_evaluate_needs_evaluation_false(
collection_schema_spec: Dict[str, Any]) -> None:
schema_loader = SchemaLoader()
collection_schema_spec['When'] = 'False'
name = schema_loader.add_schema_spec(collection_schema_spec)
schema_collection = MockBaseSchemaCollection(
name, schema_loader, AggregateSchema.ATTRIBUTE_FIELDS)
item_collection = MockBaseItemCollection(schema_collection,
EvaluationContext())
item_collection.run_evaluate()
assert item_collection.event_count == 0
def test_aggregate_persist_with_store(aggregate_schema_with_store):
aggregate = MockAggregate(schema=aggregate_schema_with_store,
identity="12345",
evaluation_context=EvaluationContext())
dt = datetime.now()
dt.replace(tzinfo=timezone.utc)
aggregate._persist(dt)
snapshot_aggregate = aggregate._store.get(
Key(identity="12345", group="user", timestamp=dt))
assert snapshot_aggregate is not None
assert snapshot_aggregate == aggregate._snapshot
def test_get_attribute(collection_schema_spec: Dict[str, Any]) -> None:
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec(collection_schema_spec)
schema_collection = MockBaseSchemaCollection(
name, schema_loader, AggregateSchema.ATTRIBUTE_FIELDS)
item_collection = MockBaseItemCollection(schema_collection,
EvaluationContext())
# Check nested items access
assert item_collection.event_count == 0
# make sure normal properties are not broken
assert item_collection._schema == schema_collection
def block_item(block_schema: BlockAggregateSchema) -> BlockAggregate:
block = BlockAggregate(block_schema, 'user1', EvaluationContext())
block.run_restore({
'events':
3,
'_start_time':
datetime(2018, 3, 7, 22, 36, 31, 0, timezone.utc).isoformat(),
'_end_time':
datetime(2018, 3, 7, 22, 37, 31, 0, timezone.utc).isoformat()
})
return block
def test_field_evaluate_implicit_typecast_integer():
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'max_attempts',
'Type': Type.INTEGER,
'Value': '23.45'
})
field_schema = IntegerFieldSchema(name, schema_loader)
field = Field(field_schema, EvaluationContext())
field.run_evaluate()
assert field._snapshot == 23
def test_field_evaluate_implicit_typecast_bool():
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'max_attempts',
'Type': Type.BOOLEAN,
'Value': '1+2'
})
field_schema = BooleanFieldSchema(name, schema_loader)
field = Field(field_schema, EvaluationContext())
field.run_evaluate()
assert field._snapshot is True
def test_evaluate_no_separation(
activity_aggregate_schema: ActivityAggregateSchema,
activity_events: List[Record]) -> None:
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
activity_aggregate = ActivityAggregate(activity_aggregate_schema, identity,
evaluation_context)
evaluation_context.global_add(activity_aggregate._schema.name,
activity_aggregate)
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 0
evaluate_event(activity_events[0], activity_aggregate)
activity_aggregate._persist()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 1
evaluate_event(activity_events[1], activity_aggregate)
activity_aggregate._persist()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 1
def test_split_when_label_evaluates_to_none(
identity_aggregate_schema_spec: Dict[str, Any],
store_spec: Dict[str, Any], records: List[Record]):
identity_aggregate_schema_spec['Dimensions'][0][
'Value'] = '1/0 if source.label == \'a\' else source.label'
schema = identity_aggregate_schema(identity_aggregate_schema_spec,
store_spec)
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
identity_aggregate = IdentityAggregate(schema, identity,
evaluation_context)
evaluation_context.global_add(identity_aggregate._schema.name,
identity_aggregate)
# Check for error states
evaluate_event(records[0], identity_aggregate)
evaluate_event(records[1], identity_aggregate)
evaluate_event(records[2], identity_aggregate)
assert identity_aggregate._dimension_fields['label'].value == 'b'
identity_aggregate.run_finalize()
store_state = identity_aggregate._store.get_all(identity)
assert len(store_state) == 1
assert store_state.get(Key('user1', 'label_aggr.b')) == {
'_identity': 'user1',
'label': 'b',
'sum': 1,
'count': 1
}
def test_field_evaluate_without_needs_evaluation():
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'max_attempts',
'Type': Type.INTEGER,
'Value': 5,
'When': '2 == 3'
})
field_schema = MockFieldSchema(name, schema_loader)
field = MockField(field_schema, EvaluationContext())
field.run_evaluate()
assert field.value == 0
def test_snapshot_invalid(collection_schema_spec: Dict[str, Any],
mock_nested_items: contextmanager) -> None:
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec(collection_schema_spec)
schema_collection = MockBaseSchemaCollection(
name, schema_loader, AggregateSchema.ATTRIBUTE_FIELDS)
# Test nested items not specified
with mock_nested_items:
with raises(SnapshotError):
item_collection = MockBaseItemCollection(schema_collection,
EvaluationContext())
assert item_collection._snapshot
def aggregate(aggregate_schema: AggregateSchema) -> Aggregate:
context = EvaluationContext()
dg = VariableAggregate(schema=aggregate_schema, identity="12345", evaluation_context=context)
context.global_add('test', dg)
context.global_add('identity', "12345")
return dg
def test_set_field_snapshot_decoding():
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'test',
'Type': Type.SET,
'Value': 'test.add(0).add(1)'
})
field_schema = SetFieldSchema(name, schema_loader)
field = Field(field_schema, EvaluationContext())
field.run_restore([2, 3])
assert field.value == {2, 3}
assert field._snapshot
assert isinstance(field._snapshot, list)
assert set(field._snapshot) == field.value
def test_set_field_snapshot_encoding():
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'test',
'Type': Type.SET,
'Value': 'test.add(0).add(1)'
})
field_schema = SetFieldSchema(name, schema_loader)
field = Field(field_schema, EvaluationContext())
field._evaluation_context.global_add('test', field.value)
field.run_evaluate()
assert field.value == {0, 1}
assert field._snapshot
assert isinstance(field._snapshot, list)
assert set(field._snapshot) == field.value
def test_aggregate_final_state(
activity_aggregate_schema: ActivityAggregateSchema,
activity_events: List[Record]) -> None:
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
activity_aggregate = ActivityAggregate(activity_aggregate_schema, identity,
evaluation_context)
evaluation_context.global_add(activity_aggregate._schema.name,
activity_aggregate)
for record in activity_events:
evaluate_event(record, activity_aggregate)
activity_aggregate.run_finalize()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 3
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 1, 1, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 1, 1, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 1, 1, 2, 1, 0,
timezone.utc).isoformat(),
'sum': 111,
'count': 3
}
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 1, 3, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 1, 3, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 1, 3, 1, 1, 0,
timezone.utc).isoformat(),
'sum': 1000,
'count': 1
}
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 2, 1, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 2, 1, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 2, 1, 1, 1, 0,
timezone.utc).isoformat(),
'sum': 10000,
'count': 1
}
def test_field_evaluate_incorrect_typecast_to_type_default(caplog):
caplog.set_level(logging.DEBUG)
schema_loader = SchemaLoader()
name = schema_loader.add_schema_spec({
'Name': 'max_attempts',
'Type': Type.INTEGER,
'Value': '"Hi"'
})
field_schema = IntegerFieldSchema(name, schema_loader)
field = Field(field_schema, EvaluationContext())
field.run_evaluate()
assert field.value == 0
assert (
'ValueError in casting Hi to Type.INTEGER for field max_attempts. Error: invalid '
'literal for int() with base 10: \'Hi\'') in caplog.records[0].message
assert caplog.records[0].levelno == logging.DEBUG
