def _prepare_window(self, start_time: datetime) -> None: """ Prepares window if any is specified. :param start_time: The anchor block start_time from where the window should be generated. """ # evaluate window first which sets the correct window in the store store = self._schema.schema_loader.get_store( self._schema.source.store_schema.fully_qualified_name) if Type.is_type_equal(self._schema.window_type, Type.DAY) or Type.is_type_equal( self._schema.window_type, Type.HOUR): block_list = self._load_blocks( store.get_range( Key(self._schema.source.key_type, self._identity, self._schema.source.name), start_time, self._get_end_time(start_time))) else: block_list = self._load_blocks( store.get_range( Key(self._schema.source.key_type, self._identity, self._schema.source.name), start_time, None, self._schema.window_value)) self._window_source = _WindowSource(block_list) self._validate_view()
def _get_end_time(self, start_time: datetime) -> datetime: """ Generates the end time to be used for the store range query. :param start_time: Start time to use as an offset to calculate the end time based on the window type in the schema. :return: """ if Type.is_type_equal(self._schema.window_type, Type.DAY): return start_time + timedelta(days=self._schema.window_value) elif Type.is_type_equal(self._schema.window_type, Type.HOUR): return start_time + timedelta(hours=self._schema.window_value)
def test_base_item_valid(schema_spec: Dict[str, Any]) -> None: test_item = get_test_item(schema_spec) assert test_item._schema.name == 'TestField' assert Type.is_type_equal(test_item._schema.type, Type.INTEGER) assert len(test_item._evaluation_context.global_context) == 0 assert len(test_item._evaluation_context.local_context) == 0 assert test_item._needs_evaluation
def test_window_transformer_schema_init(schema_loader, stream_schema_spec, window_schema_spec): schema_loader.add_schema_spec(stream_schema_spec) window_dtc_name = schema_loader.add_schema_spec(window_schema_spec) window_transformer_schema = WindowTransformerSchema(window_dtc_name, schema_loader) anchor_spec = schema_loader.get_schema_spec('ProductMLExample.anchor') assert anchor_spec == window_schema_spec['Anchor'] assert anchor_spec['Name'] == 'anchor' assert Type.is_type_equal(anchor_spec['Type'], Type.ANCHOR) assert isinstance(window_transformer_schema.anchor, AnchorSchema)
def _validate_view(self): if Type.is_type_equal(self._schema.window_type, Type.COUNT) and len( self._window_source.view) != abs(self._schema.window_value): raise PrepareWindowMissingBlocksError( '{} WindowAggregate: Expecting {} but found {} blocks'.format( self._schema.name, abs(self._schema.window_value), len(self._window_source.view))) if len(self._window_source.view) == 0: raise PrepareWindowMissingBlocksError( '{} WindowAggregate: No matching blocks found'.format( self._schema.name))
def test_initialization_with_valid_source( schema_loader_with_mem_store: SchemaLoader, aggregate_block_schema_spec: Dict[str, Any], window_schema_spec: Dict[str, Any], stream_dtc_name: str): schema_loader_with_mem_store.add_schema_spec(aggregate_block_schema_spec, stream_dtc_name) name = schema_loader_with_mem_store.add_schema_spec(window_schema_spec) window_aggregate_schema = WindowAggregateSchema( name, schema_loader_with_mem_store) assert Type.is_type_equal(window_aggregate_schema.window_type, Type.DAY) assert window_aggregate_schema.window_value == 1 assert isinstance(window_aggregate_schema.source, BlockAggregateSchema) assert window_aggregate_schema.source.name == 'session'
def is_streaming_dtc(dtc_dict: Dict) -> bool: return Type.is_type_equal(dtc_dict.get('Type', ''), Type.BLURR_TRANSFORM_STREAMING)
def is_window_dtc(dtc_dict: Dict) -> bool: return Type.is_type_equal(dtc_dict.get('Type', ''), Type.BLURR_TRANSFORM_WINDOW)
def test_is_type_equal_returns_true_when_actual_type_is_equal_type(): assert Type.is_type_equal(Type.DAY, Type.DAY)
def test_is_type_equal_returns_true_when_actual_type_is_equal_string(): assert Type.is_type_equal("day", Type.DAY)
def test_is_type_equal_returns_false_when_actual_type_is_invalid_string(): assert Type.is_type_equal("invalid", Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_type(): assert Type.is_type_equal(Type.DAY, Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_string(): assert Type.is_type_equal("hour", Type.DAY) is False