def _execute_window_dtc(self, identity: str, schema_loader: SchemaLoader) -> List[Dict]: if self._window_dtc is None: logging.debug('Window DTC not provided') return [] stream_transformer = StreamingTransformer( self._get_streaming_transformer_schema(schema_loader), identity) all_data = self._get_store(schema_loader).get_all(identity) stream_transformer.run_restore(all_data) exec_context = Context() exec_context.add(stream_transformer._schema.name, stream_transformer) block_obj = None for aggregate in stream_transformer._nested_items.values(): if not isinstance(aggregate, BlockAggregate): continue if block_obj is not None: raise Exception( ('Window operation is supported against Streaming ', 'DTC with only one BlockAggregate')) block_obj = aggregate if block_obj is None: raise Exception( 'No BlockAggregate found in the Streaming DTC file') window_data = [] window_dtc_name = schema_loader.add_schema_spec(self._window_dtc) window_transformer_schema = schema_loader.get_schema_object( window_dtc_name) window_transformer = WindowTransformer(window_transformer_schema, identity, exec_context) logging.debug('Running Window DTC for identity {}'.format(identity)) anchors = 0 blocks = 0 for key, data in all_data.items(): if key.group != block_obj._schema.name: continue try: blocks += 1 if window_transformer.run_evaluate( block_obj.run_restore(data)): anchors += 1 window_data.append( window_transformer.run_flattened_snapshot) except PrepareWindowMissingBlocksError as err: logging.debug('{} with {}'.format(err, key)) if anchors == 0: logging.debug( 'No anchors found for identity {} out of {} blocks'.format( identity, blocks)) return window_data
def test_expression_globals_locals() -> None: code_string = 'a + b + 1' expr = Expression(code_string) with pytest.raises(NameError, match='name \'a\' is not defined'): expr.evaluate(EvaluationContext()) assert expr.evaluate(EvaluationContext(Context({'a': 2, 'b': 3}))) == 6 assert expr.evaluate( EvaluationContext(local_context=Context({ 'a': 2, 'b': 3 }))) == 6 assert expr.evaluate( EvaluationContext(Context({'a': 2}), Context({'b': 3}))) == 6
def window_transformer(schema_loader, stream_transformer, window_schema_spec): window_dtc_name = schema_loader.add_schema_spec(window_schema_spec) return WindowTransformer( schema_loader.get_schema_object(window_dtc_name), 'user1', Context({ stream_transformer._schema.name: stream_transformer }))
def test_streaming_transformer_schema_get_time_type_error(schema_loader: SchemaLoader, schema_spec: Dict[str, Any]) -> None: schema_spec['Time'] = '1' streaming_bts = schema_loader.add_schema_spec(schema_spec) transformer_schema = StreamingTransformerSchema(streaming_bts, schema_loader) with pytest.raises(TimeError, match='Could not determine time using 1'): assert transformer_schema.get_time(Context())
def test_streaming_transformer_schema_get_time_datetime_not_defined( schema_loader: SchemaLoader, schema_spec: Dict[str, Any]) -> None: del schema_spec['Import'] streaming_bts = schema_loader.add_schema_spec(schema_spec) transformer_schema = StreamingTransformerSchema(streaming_bts, schema_loader) with pytest.raises(NameError, match='name \'datetime\' is not defined'): assert transformer_schema.get_time(Context())
def test_execution_error_type_mismatch(caplog) -> None: caplog.set_level(logging.DEBUG) code_string = '1 + \'a\'' assert Expression(code_string).evaluate( EvaluationContext(Context({'test_dict': {}}))) is None assert 'TypeError in evaluating expression 1 + \'a\'' in caplog.records[ 0].message assert caplog.records[0].levelno == logging.DEBUG
def test_execution_key_error(caplog) -> None: caplog.set_level(logging.DEBUG) code_string = 'test_dict[\'missing_key\'] + 1' assert Expression(code_string).evaluate( EvaluationContext(Context({'test_dict': {}}))) is None assert 'KeyError in evaluating expression test_dict[\'missing_key\'] + 1. Error: \'missing_key\'' in caplog.records[ 0].message assert caplog.records[0].levelno == logging.DEBUG
def test_expression_user_function() -> None: code_string = '2 if test_function() else 3' def test_function(): return 3 > 4 expr = Expression(code_string) assert expr.evaluate( EvaluationContext(Context({'test_function': test_function}))) == 3
def test_execution_error_missing_field(caplog, schema_loader: SchemaLoader) -> None: caplog.set_level(logging.DEBUG) context = Context({ 'test': StreamingTransformer(schema_loader.get_schema_object('test'), 'user1') }) with raises(MissingAttributeError, match='missing_field not defined in test_group'): Expression('test.test_group.missing_field').evaluate( EvaluationContext(context)) assert ( 'MissingAttributeError in evaluating expression test.test_group.missing_field. ' 'Error: missing_field not defined in test_group') in caplog.text with raises(MissingAttributeError, match='missing_field not defined in test_group'): Expression('test.test_group[\'missing_field\']').evaluate( EvaluationContext(context))
def window_aggregate( window_aggregate_schema: WindowAggregateSchema) -> WindowAggregate: return WindowAggregate(window_aggregate_schema, "user1", EvaluationContext(Context({'identity': 'user1'})))
def test_streaming_transformer_schema_get_identity_constant( schema_loader: SchemaLoader, schema_spec: Dict[str, Any]) -> None: streaming_dtc = schema_loader.add_schema_spec(schema_spec) transformer_schema = StreamingTransformerSchema(streaming_dtc, schema_loader) assert transformer_schema.get_identity(Context()) == 'user1'