def _execute_window_dtc(self, identity: str,
schema_loader: SchemaLoader) -> List[Dict]:
if self._window_dtc is None:
logging.debug('Window DTC not provided')
return []
stream_transformer = StreamingTransformer(
self._get_streaming_transformer_schema(schema_loader), identity)
all_data = self._get_store(schema_loader).get_all(identity)
stream_transformer.run_restore(all_data)
exec_context = Context()
exec_context.add(stream_transformer._schema.name, stream_transformer)
block_obj = None
for aggregate in stream_transformer._nested_items.values():
if not isinstance(aggregate, BlockAggregate):
continue
if block_obj is not None:
raise Exception(
('Window operation is supported against Streaming ',
'DTC with only one BlockAggregate'))
block_obj = aggregate
if block_obj is None:
raise Exception(
'No BlockAggregate found in the Streaming DTC file')
window_data = []
window_dtc_name = schema_loader.add_schema_spec(self._window_dtc)
window_transformer_schema = schema_loader.get_schema_object(
window_dtc_name)
window_transformer = WindowTransformer(window_transformer_schema,
identity, exec_context)
logging.debug('Running Window DTC for identity {}'.format(identity))
anchors = 0
blocks = 0
for key, data in all_data.items():
if key.group != block_obj._schema.name:
continue
try:
blocks += 1
if window_transformer.run_evaluate(
block_obj.run_restore(data)):
anchors += 1
window_data.append(
window_transformer.run_flattened_snapshot)
except PrepareWindowMissingBlocksError as err:
logging.debug('{} with {}'.format(err, key))
if anchors == 0:
logging.debug(
'No anchors found for identity {} out of {} blocks'.format(
identity, blocks))
return window_data
def test_expression_globals_locals() -> None:
code_string = 'a + b + 1'
expr = Expression(code_string)
with pytest.raises(NameError, match='name \'a\' is not defined'):
expr.evaluate(EvaluationContext())
assert expr.evaluate(EvaluationContext(Context({'a': 2, 'b': 3}))) == 6
assert expr.evaluate(
EvaluationContext(local_context=Context({
'a': 2,
'b': 3
}))) == 6
assert expr.evaluate(
EvaluationContext(Context({'a': 2}), Context({'b': 3}))) == 6
def window_transformer(schema_loader, stream_transformer, window_schema_spec):
window_dtc_name = schema_loader.add_schema_spec(window_schema_spec)
return WindowTransformer(
schema_loader.get_schema_object(window_dtc_name), 'user1',
Context({
stream_transformer._schema.name: stream_transformer
}))
def test_streaming_transformer_schema_get_time_type_error(schema_loader: SchemaLoader,
schema_spec: Dict[str, Any]) -> None:
schema_spec['Time'] = '1'
streaming_bts = schema_loader.add_schema_spec(schema_spec)
transformer_schema = StreamingTransformerSchema(streaming_bts, schema_loader)
with pytest.raises(TimeError, match='Could not determine time using 1'):
assert transformer_schema.get_time(Context())
def test_streaming_transformer_schema_get_time_datetime_not_defined(
schema_loader: SchemaLoader, schema_spec: Dict[str, Any]) -> None:
del schema_spec['Import']
streaming_bts = schema_loader.add_schema_spec(schema_spec)
transformer_schema = StreamingTransformerSchema(streaming_bts, schema_loader)
with pytest.raises(NameError, match='name \'datetime\' is not defined'):
assert transformer_schema.get_time(Context())
def test_execution_error_type_mismatch(caplog) -> None:
caplog.set_level(logging.DEBUG)
code_string = '1 + \'a\''
assert Expression(code_string).evaluate(
EvaluationContext(Context({'test_dict': {}}))) is None
assert 'TypeError in evaluating expression 1 + \'a\'' in caplog.records[
0].message
assert caplog.records[0].levelno == logging.DEBUG
def test_execution_key_error(caplog) -> None:
caplog.set_level(logging.DEBUG)
code_string = 'test_dict[\'missing_key\'] + 1'
assert Expression(code_string).evaluate(
EvaluationContext(Context({'test_dict': {}}))) is None
assert 'KeyError in evaluating expression test_dict[\'missing_key\'] + 1. Error: \'missing_key\'' in caplog.records[
0].message
assert caplog.records[0].levelno == logging.DEBUG
def test_expression_user_function() -> None:
code_string = '2 if test_function() else 3'
def test_function():
return 3 > 4
expr = Expression(code_string)
assert expr.evaluate(
EvaluationContext(Context({'test_function': test_function}))) == 3
def test_execution_error_missing_field(caplog,
schema_loader: SchemaLoader) -> None:
caplog.set_level(logging.DEBUG)
context = Context({
'test':
StreamingTransformer(schema_loader.get_schema_object('test'), 'user1')
})
with raises(MissingAttributeError,
match='missing_field not defined in test_group'):
Expression('test.test_group.missing_field').evaluate(
EvaluationContext(context))
assert (
'MissingAttributeError in evaluating expression test.test_group.missing_field. '
'Error: missing_field not defined in test_group') in caplog.text
with raises(MissingAttributeError,
match='missing_field not defined in test_group'):
Expression('test.test_group[\'missing_field\']').evaluate(
EvaluationContext(context))
def window_aggregate(
window_aggregate_schema: WindowAggregateSchema) -> WindowAggregate:
return WindowAggregate(window_aggregate_schema, "user1",
EvaluationContext(Context({'identity': 'user1'})))
def test_streaming_transformer_schema_get_identity_constant(
schema_loader: SchemaLoader, schema_spec: Dict[str, Any]) -> None:
streaming_dtc = schema_loader.add_schema_spec(schema_spec)
transformer_schema = StreamingTransformerSchema(streaming_dtc,
schema_loader)
assert transformer_schema.get_identity(Context()) == 'user1'