def test_split_when_label_evaluates_to_none(
identity_aggregate_schema_spec: Dict[str, Any],
store_spec: Dict[str, Any], records: List[Record]):
identity_aggregate_schema_spec['Dimensions'][0][
'Value'] = '1/0 if source.label == \'a\' else source.label'
schema = identity_aggregate_schema(identity_aggregate_schema_spec,
store_spec)
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
identity_aggregate = IdentityAggregate(schema, identity,
evaluation_context)
evaluation_context.global_add(identity_aggregate._schema.name,
identity_aggregate)
# Check for error states
evaluate_event(records[0], identity_aggregate)
evaluate_event(records[1], identity_aggregate)
evaluate_event(records[2], identity_aggregate)
assert identity_aggregate._dimension_fields['label'].value == 'b'
identity_aggregate.run_finalize()
store_state = identity_aggregate._store.get_all(identity)
assert len(store_state) == 1
assert store_state.get(Key('user1', 'label_aggr.b')) == {
'_identity': 'user1',
'label': 'b',
'sum': 1,
'count': 1
}
def test_evaluate_no_separation(
activity_aggregate_schema: ActivityAggregateSchema,
activity_events: List[Record]) -> None:
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
activity_aggregate = ActivityAggregate(activity_aggregate_schema, identity,
evaluation_context)
evaluation_context.global_add(activity_aggregate._schema.name,
activity_aggregate)
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 0
evaluate_event(activity_events[0], activity_aggregate)
activity_aggregate._persist()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 1
evaluate_event(activity_events[1], activity_aggregate)
activity_aggregate._persist()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 1
def aggregate(aggregate_schema: AggregateSchema) -> Aggregate:
context = EvaluationContext()
dg = VariableAggregate(schema=aggregate_schema, identity="12345", evaluation_context=context)
context.global_add('test', dg)
context.global_add('identity', "12345")
return dg
def test_aggregate_final_state(
activity_aggregate_schema: ActivityAggregateSchema,
activity_events: List[Record]) -> None:
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
activity_aggregate = ActivityAggregate(activity_aggregate_schema, identity,
evaluation_context)
evaluation_context.global_add(activity_aggregate._schema.name,
activity_aggregate)
for record in activity_events:
evaluate_event(record, activity_aggregate)
activity_aggregate.run_finalize()
store_state = activity_aggregate._store.get_all(identity)
assert len(store_state) == 3
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 1, 1, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 1, 1, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 1, 1, 2, 1, 0,
timezone.utc).isoformat(),
'sum': 111,
'count': 3
}
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 1, 3, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 1, 3, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 1, 3, 1, 1, 0,
timezone.utc).isoformat(),
'sum': 1000,
'count': 1
}
assert store_state.get(
Key('user1', 'activity_aggr',
datetime(2018, 1, 2, 1, 1, 1, 0, timezone.utc))) == {
'_identity': 'user1',
'_start_time': datetime(2018, 1, 2, 1, 1, 1, 0,
timezone.utc).isoformat(),
'_end_time': datetime(2018, 1, 2, 1, 1, 1, 0,
timezone.utc).isoformat(),
'sum': 10000,
'count': 1
}
def test_two_key_fields_in_aggregate(
identity_aggregate_schema_spec_with_two_key_fields: Dict[str, Any],
store_spec: Dict[str, Any], records: List[Record]):
schema = identity_aggregate_schema(
identity_aggregate_schema_spec_with_two_key_fields, store_spec)
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
identity_aggregate = IdentityAggregate(schema, identity,
evaluation_context)
evaluation_context.global_add(identity_aggregate._schema.name,
identity_aggregate)
# Evaluate all the events
for event in records:
evaluate_event(event, identity_aggregate)
identity_aggregate.run_finalize()
store_state = identity_aggregate._store.get_all('user1')
assert len(store_state) == 3
assert store_state.get(
Key(KeyType.DIMENSION, 'user1', 'label_aggr', ['a', '97'])) == {
'_identity': 'user1',
'label': 'a',
'label_ascii': 97,
'sum': 110,
'count': 2
}
assert store_state.get(
Key(KeyType.DIMENSION, 'user1', 'label_aggr', ['b', '98'])) == {
'_identity': 'user1',
'label': 'b',
'label_ascii': 98,
'sum': 1,
'count': 1
}
assert store_state.get(
Key(KeyType.DIMENSION, 'user1', 'label_aggr', ['c', '99'])) == {
'_identity': 'user1',
'label': 'c',
'label_ascii': 99,
'sum': 11000,
'count': 2
}
def create_block_aggregate(schema, time, identity) -> BlockAggregate:
evaluation_context = EvaluationContext()
block_aggregate = BlockAggregate(schema=schema,
identity=identity,
evaluation_context=evaluation_context)
evaluation_context.global_add('time', time)
evaluation_context.global_add('user', block_aggregate)
evaluation_context.global_add('identity', identity)
return block_aggregate
def test_split_by_label_valid(identity_aggregate_schema_spec: Dict[str, Any],
store_spec: Dict[str,
Any], records: List[Record]):
schema = identity_aggregate_schema(identity_aggregate_schema_spec,
store_spec)
# Initialize the starting state
identity = 'user1'
evaluation_context = EvaluationContext()
evaluation_context.global_add('identity', identity)
identity_aggregate = IdentityAggregate(schema, identity,
evaluation_context)
evaluation_context.global_add(identity_aggregate._schema.name,
identity_aggregate)
# Check that initial state is empty
assert identity_aggregate._dimension_fields['label'].value == ''
assert identity_aggregate._store.get_all(identity) == {}
# Check state at the end of the first event processed
evaluate_event(records[0], identity_aggregate)
assert identity_aggregate._dimension_fields['label'].value == 'a'
assert identity_aggregate._store.get_all(identity) == {}
# Check for labeled partition and persistence of the first label when label changes
evaluate_event(records[1], identity_aggregate)
assert identity_aggregate._dimension_fields['label'].value == 'b'
evaluate_event(records[2], identity_aggregate)
identity_aggregate._persist()
store_state = identity_aggregate._store.get_all(identity)
assert identity_aggregate._dimension_fields['label'].value == 'a'
assert len(store_state) == 2
# Check for final state
evaluate_event(records[3], identity_aggregate)
evaluate_event(records[4], identity_aggregate)
identity_aggregate._persist()
assert identity_aggregate._dimension_fields['label'].value == 'c'
store_state = identity_aggregate._store.get_all(identity)
assert len(store_state) == 3
assert store_state.get(Key('user1', 'label_aggr.a')) == {
'_identity': 'user1',
'label': 'a',
'sum': 110,
'count': 2
}
assert store_state.get(Key('user1', 'label_aggr.b')) == {
'_identity': 'user1',
'label': 'b',
'sum': 1,
'count': 1
}
assert store_state.get(Key('user1', 'label_aggr.c')) == {
'_identity': 'user1',
'label': 'c',
'sum': 11000,
'count': 2
}