def _prepare_window(self, start_time: datetime) -> None:
"""
Prepares window if any is specified.
:param start_time: The anchor block start_time from where the window
should be generated.
"""
# evaluate window first which sets the correct window in the store
store = self._schema.schema_loader.get_store(
self._schema.source.store_schema.fully_qualified_name)
if Type.is_type_equal(self._schema.window_type,
Type.DAY) or Type.is_type_equal(
self._schema.window_type, Type.HOUR):
block_list = self._load_blocks(
store.get_range(
Key(self._schema.source.key_type, self._identity,
self._schema.source.name), start_time,
self._get_end_time(start_time)))
else:
block_list = self._load_blocks(
store.get_range(
Key(self._schema.source.key_type, self._identity,
self._schema.source.name), start_time, None,
self._schema.window_value))
self._window_source = _WindowSource(block_list)
self._validate_view()
def _get_end_time(self, start_time: datetime) -> datetime:
"""
Generates the end time to be used for the store range query.
:param start_time: Start time to use as an offset to calculate the end time
based on the window type in the schema.
:return:
"""
if Type.is_type_equal(self._schema.window_type, Type.DAY):
return start_time + timedelta(days=self._schema.window_value)
elif Type.is_type_equal(self._schema.window_type, Type.HOUR):
return start_time + timedelta(hours=self._schema.window_value)
def test_base_item_valid(schema_spec: Dict[str, Any]) -> None:
test_item = get_test_item(schema_spec)
assert test_item._schema.name == 'TestField'
assert Type.is_type_equal(test_item._schema.type, Type.INTEGER)
assert len(test_item._evaluation_context.global_context) == 0
assert len(test_item._evaluation_context.local_context) == 0
assert test_item._needs_evaluation
def test_window_transformer_schema_init(schema_loader, stream_schema_spec, window_schema_spec):
schema_loader.add_schema_spec(stream_schema_spec)
window_dtc_name = schema_loader.add_schema_spec(window_schema_spec)
window_transformer_schema = WindowTransformerSchema(window_dtc_name, schema_loader)
anchor_spec = schema_loader.get_schema_spec('ProductMLExample.anchor')
assert anchor_spec == window_schema_spec['Anchor']
assert anchor_spec['Name'] == 'anchor'
assert Type.is_type_equal(anchor_spec['Type'], Type.ANCHOR)
assert isinstance(window_transformer_schema.anchor, AnchorSchema)
def _validate_view(self):
if Type.is_type_equal(self._schema.window_type, Type.COUNT) and len(
self._window_source.view) != abs(self._schema.window_value):
raise PrepareWindowMissingBlocksError(
'{} WindowAggregate: Expecting {} but found {} blocks'.format(
self._schema.name, abs(self._schema.window_value),
len(self._window_source.view)))
if len(self._window_source.view) == 0:
raise PrepareWindowMissingBlocksError(
'{} WindowAggregate: No matching blocks found'.format(
self._schema.name))
def test_initialization_with_valid_source(
schema_loader_with_mem_store: SchemaLoader,
aggregate_block_schema_spec: Dict[str, Any],
window_schema_spec: Dict[str, Any], stream_dtc_name: str):
schema_loader_with_mem_store.add_schema_spec(aggregate_block_schema_spec,
stream_dtc_name)
name = schema_loader_with_mem_store.add_schema_spec(window_schema_spec)
window_aggregate_schema = WindowAggregateSchema(
name, schema_loader_with_mem_store)
assert Type.is_type_equal(window_aggregate_schema.window_type, Type.DAY)
assert window_aggregate_schema.window_value == 1
assert isinstance(window_aggregate_schema.source, BlockAggregateSchema)
assert window_aggregate_schema.source.name == 'session'
def is_streaming_dtc(dtc_dict: Dict) -> bool:
return Type.is_type_equal(dtc_dict.get('Type', ''),
Type.BLURR_TRANSFORM_STREAMING)
def is_window_dtc(dtc_dict: Dict) -> bool:
return Type.is_type_equal(dtc_dict.get('Type', ''),
Type.BLURR_TRANSFORM_WINDOW)
def test_is_type_equal_returns_true_when_actual_type_is_equal_type():
assert Type.is_type_equal(Type.DAY, Type.DAY)
def test_is_type_equal_returns_true_when_actual_type_is_equal_string():
assert Type.is_type_equal("day", Type.DAY)
def test_is_type_equal_returns_false_when_actual_type_is_invalid_string():
assert Type.is_type_equal("invalid", Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_type():
assert Type.is_type_equal(Type.DAY, Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_string():
assert Type.is_type_equal("hour", Type.DAY) is False
