def _prepare_window(self, start_time: datetime) -> None:
"""
Prepares window if any is specified.
:param start_time: The anchor block start_time from where the window
should be generated.
"""
# evaluate window first which sets the correct window in the store
store = self._schema.schema_loader.get_store(
self._schema.source.store_schema.fully_qualified_name)
if Type.is_type_equal(self._schema.window_type,
Type.DAY) or Type.is_type_equal(
self._schema.window_type, Type.HOUR):
block_list = self._load_blocks(
store.get_range(
Key(self._schema.source.key_type, self._identity,
self._schema.source.name), start_time,
self._get_end_time(start_time)))
else:
block_list = self._load_blocks(
store.get_range(
Key(self._schema.source.key_type, self._identity,
self._schema.source.name), start_time, None,
self._schema.window_value))
self._window_source = _WindowSource(block_list)
self._validate_view()
def _get_end_time(self, start_time: datetime) -> datetime:
"""
Generates the end time to be used for the store range query.
:param start_time: Start time to use as an offset to calculate the end time
based on the window type in the schema.
:return:
"""
if Type.is_type_equal(self._schema.window_type, Type.DAY):
return start_time + timedelta(days=self._schema.window_value)
elif Type.is_type_equal(self._schema.window_type, Type.HOUR):
return start_time + timedelta(hours=self._schema.window_value)
def load_type(type_name: Union[str, Type], type_map: dict) -> Any:
try:
type_name_enum = Type(type_name)
return TypeLoader.import_class_by_full_name(
type_map[type_name_enum])
except (KeyError, ValueError):
raise GenericSchemaError('Type `{}` not found.'.format(type_name))
def test_base_item_valid(schema_spec: Dict[str, Any]) -> None:
test_item = get_test_item(schema_spec)
assert test_item._schema.name == 'TestField'
assert Type.is_type_equal(test_item._schema.type, Type.INTEGER)
assert len(test_item._evaluation_context.global_context) == 0
assert len(test_item._evaluation_context.local_context) == 0
assert test_item._needs_evaluation
def load_type(type_name: Union[str, Type], type_map: dict) -> Any:
type_class = None
try:
type_name_enum = Type(type_name)
type_class = type_map[type_name_enum]
return TypeLoader.import_class_by_full_name(
type_map[type_name_enum])
except (KeyError, ValueError):
raise TypeLoaderError(str(type_name), type_class)
def test_window_transformer_schema_init(schema_loader, stream_schema_spec, window_schema_spec):
schema_loader.add_schema_spec(stream_schema_spec)
window_dtc_name = schema_loader.add_schema_spec(window_schema_spec)
window_transformer_schema = WindowTransformerSchema(window_dtc_name, schema_loader)
anchor_spec = schema_loader.get_schema_spec('ProductMLExample.anchor')
assert anchor_spec == window_schema_spec['Anchor']
assert anchor_spec['Name'] == 'anchor'
assert Type.is_type_equal(anchor_spec['Type'], Type.ANCHOR)
assert isinstance(window_transformer_schema.anchor, AnchorSchema)
def _validate_view(self):
if Type.is_type_equal(self._schema.window_type, Type.COUNT) and len(
self._window_source.view) != abs(self._schema.window_value):
raise PrepareWindowMissingBlocksError(
'{} WindowAggregate: Expecting {} but found {} blocks'.format(
self._schema.name, abs(self._schema.window_value),
len(self._window_source.view)))
if len(self._window_source.view) == 0:
raise PrepareWindowMissingBlocksError(
'{} WindowAggregate: No matching blocks found'.format(
self._schema.name))
def test_initialization_with_valid_source(
schema_loader_with_mem_store: SchemaLoader,
aggregate_block_schema_spec: Dict[str, Any],
window_schema_spec: Dict[str, Any], stream_dtc_name: str):
schema_loader_with_mem_store.add_schema_spec(aggregate_block_schema_spec,
stream_dtc_name)
name = schema_loader_with_mem_store.add_schema_spec(window_schema_spec)
window_aggregate_schema = WindowAggregateSchema(
name, schema_loader_with_mem_store)
assert Type.is_type_equal(window_aggregate_schema.window_type, Type.DAY)
assert window_aggregate_schema.window_value == 1
assert isinstance(window_aggregate_schema.source, BlockAggregateSchema)
assert window_aggregate_schema.source.name == 'session'
def get_schema_specs_of_type(
self, *schema_types: Type) -> Dict[str, Dict[str, Any]]:
"""
Returns a list of fully qualified names and schema dictionary tuples for
the schema types provided.
:param schema_types: Schema types.
:return: List of fully qualified names and schema dictionary tuples.
"""
return {
fq_name: schema
for fq_name, schema in self._spec_cache.items() if Type.is_type_in(
schema.get(ATTRIBUTE_TYPE, ''), list(schema_types))
}
def add_schema_spec(
self,
spec: Dict[str, Any],
fully_qualified_parent_name: str = None) -> Optional[str]:
"""
Add a schema dictionary to the schema loader. The given schema is stored
against fully_qualified_parent_name + ITEM_SEPARATOR('.') + schema.name.
:param spec: Schema specification.
:param fully_qualified_parent_name: Full qualified name of the parent.
If None is passed then the schema is stored against the schema name.
:return: The fully qualified name against which the spec is stored.
None is returned if the given spec is not a dictionary or the spec does not
contain a 'name' key.
"""
if not isinstance(spec, dict) or ATTRIBUTE_NAME not in spec:
return None
name = spec[ATTRIBUTE_NAME]
fully_qualified_name = name if fully_qualified_parent_name is None else self.get_fully_qualified_name(
fully_qualified_parent_name, name)
# Ensure that basic validation for each spec part is done before it is added to spec cache
if isinstance(spec, dict):
self._error_cache.add(
validate_required_attributes(fully_qualified_name, spec,
ATTRIBUTE_NAME, ATTRIBUTE_TYPE))
if ATTRIBUTE_TYPE in spec and not Type.contains(
spec[ATTRIBUTE_TYPE]):
self._error_cache.add(
InvalidTypeError(fully_qualified_name, spec,
ATTRIBUTE_TYPE,
InvalidTypeError.Reason.TYPE_NOT_DEFINED))
self._spec_cache[fully_qualified_name] = spec
for key, val in spec.items():
if isinstance(val, list):
for item in val:
self.add_schema_spec(item, fully_qualified_name)
self.add_schema_spec(val, fully_qualified_name)
return spec[ATTRIBUTE_NAME]
def get_store(self, fully_qualified_name: str) -> Optional['Store']:
"""
Used to generate a store object from the given fully_qualified_name.
:param fully_qualified_name: The fully qualified name of the store object needed.
:return: An initialized store object
"""
if fully_qualified_name not in self._store_cache:
schema = self.get_schema_object(fully_qualified_name)
if not schema:
return None
if Type.is_store_type(schema.type):
self._store_cache[fully_qualified_name] = TypeLoader.load_item(
schema.type)(schema)
else:
self.add_errors(
InvalidTypeError(fully_qualified_name, {}, ATTRIBUTE_TYPE,
InvalidTypeError.Reason.INCORRECT_BASE,
schema.type,
InvalidTypeError.BaseTypes.STORE))
return self._store_cache.get(fully_qualified_name, None)
def test_is_type_equal_returns_true_when_actual_type_is_equal_string():
assert Type.is_type_equal("day", Type.DAY)
def test_is_type_equal_returns_false_when_actual_type_is_invalid_string():
assert Type.is_type_equal("invalid", Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_type():
assert Type.is_type_equal(Type.DAY, Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_string():
assert Type.is_type_equal("hour", Type.DAY) is False
def is_window_dtc(dtc_dict: Dict) -> bool:
return Type.is_type_equal(dtc_dict.get('Type', ''),
Type.BLURR_TRANSFORM_WINDOW)
def is_streaming_dtc(dtc_dict: Dict) -> bool:
return Type.is_type_equal(dtc_dict.get('Type', ''),
Type.BLURR_TRANSFORM_STREAMING)
def test_is_type_equal_returns_true_when_actual_type_is_equal_type():
assert Type.is_type_equal(Type.DAY, Type.DAY)
