def _prepare_window(self, start_time: datetime) -> None: """ Prepares window if any is specified. :param start_time: The anchor block start_time from where the window should be generated. """ # evaluate window first which sets the correct window in the store store = self._schema.schema_loader.get_store( self._schema.source.store_schema.fully_qualified_name) if Type.is_type_equal(self._schema.window_type, Type.DAY) or Type.is_type_equal( self._schema.window_type, Type.HOUR): block_list = self._load_blocks( store.get_range( Key(self._schema.source.key_type, self._identity, self._schema.source.name), start_time, self._get_end_time(start_time))) else: block_list = self._load_blocks( store.get_range( Key(self._schema.source.key_type, self._identity, self._schema.source.name), start_time, None, self._schema.window_value)) self._window_source = _WindowSource(block_list) self._validate_view()
def _get_end_time(self, start_time: datetime) -> datetime: """ Generates the end time to be used for the store range query. :param start_time: Start time to use as an offset to calculate the end time based on the window type in the schema. :return: """ if Type.is_type_equal(self._schema.window_type, Type.DAY): return start_time + timedelta(days=self._schema.window_value) elif Type.is_type_equal(self._schema.window_type, Type.HOUR): return start_time + timedelta(hours=self._schema.window_value)
def load_type(type_name: Union[str, Type], type_map: dict) -> Any: try: type_name_enum = Type(type_name) return TypeLoader.import_class_by_full_name( type_map[type_name_enum]) except (KeyError, ValueError): raise GenericSchemaError('Type `{}` not found.'.format(type_name))
def test_base_item_valid(schema_spec: Dict[str, Any]) -> None: test_item = get_test_item(schema_spec) assert test_item._schema.name == 'TestField' assert Type.is_type_equal(test_item._schema.type, Type.INTEGER) assert len(test_item._evaluation_context.global_context) == 0 assert len(test_item._evaluation_context.local_context) == 0 assert test_item._needs_evaluation
def load_type(type_name: Union[str, Type], type_map: dict) -> Any: type_class = None try: type_name_enum = Type(type_name) type_class = type_map[type_name_enum] return TypeLoader.import_class_by_full_name( type_map[type_name_enum]) except (KeyError, ValueError): raise TypeLoaderError(str(type_name), type_class)
def test_window_transformer_schema_init(schema_loader, stream_schema_spec, window_schema_spec): schema_loader.add_schema_spec(stream_schema_spec) window_dtc_name = schema_loader.add_schema_spec(window_schema_spec) window_transformer_schema = WindowTransformerSchema(window_dtc_name, schema_loader) anchor_spec = schema_loader.get_schema_spec('ProductMLExample.anchor') assert anchor_spec == window_schema_spec['Anchor'] assert anchor_spec['Name'] == 'anchor' assert Type.is_type_equal(anchor_spec['Type'], Type.ANCHOR) assert isinstance(window_transformer_schema.anchor, AnchorSchema)
def _validate_view(self): if Type.is_type_equal(self._schema.window_type, Type.COUNT) and len( self._window_source.view) != abs(self._schema.window_value): raise PrepareWindowMissingBlocksError( '{} WindowAggregate: Expecting {} but found {} blocks'.format( self._schema.name, abs(self._schema.window_value), len(self._window_source.view))) if len(self._window_source.view) == 0: raise PrepareWindowMissingBlocksError( '{} WindowAggregate: No matching blocks found'.format( self._schema.name))
def test_initialization_with_valid_source( schema_loader_with_mem_store: SchemaLoader, aggregate_block_schema_spec: Dict[str, Any], window_schema_spec: Dict[str, Any], stream_dtc_name: str): schema_loader_with_mem_store.add_schema_spec(aggregate_block_schema_spec, stream_dtc_name) name = schema_loader_with_mem_store.add_schema_spec(window_schema_spec) window_aggregate_schema = WindowAggregateSchema( name, schema_loader_with_mem_store) assert Type.is_type_equal(window_aggregate_schema.window_type, Type.DAY) assert window_aggregate_schema.window_value == 1 assert isinstance(window_aggregate_schema.source, BlockAggregateSchema) assert window_aggregate_schema.source.name == 'session'
def get_schema_specs_of_type( self, *schema_types: Type) -> Dict[str, Dict[str, Any]]: """ Returns a list of fully qualified names and schema dictionary tuples for the schema types provided. :param schema_types: Schema types. :return: List of fully qualified names and schema dictionary tuples. """ return { fq_name: schema for fq_name, schema in self._spec_cache.items() if Type.is_type_in( schema.get(ATTRIBUTE_TYPE, ''), list(schema_types)) }
def add_schema_spec( self, spec: Dict[str, Any], fully_qualified_parent_name: str = None) -> Optional[str]: """ Add a schema dictionary to the schema loader. The given schema is stored against fully_qualified_parent_name + ITEM_SEPARATOR('.') + schema.name. :param spec: Schema specification. :param fully_qualified_parent_name: Full qualified name of the parent. If None is passed then the schema is stored against the schema name. :return: The fully qualified name against which the spec is stored. None is returned if the given spec is not a dictionary or the spec does not contain a 'name' key. """ if not isinstance(spec, dict) or ATTRIBUTE_NAME not in spec: return None name = spec[ATTRIBUTE_NAME] fully_qualified_name = name if fully_qualified_parent_name is None else self.get_fully_qualified_name( fully_qualified_parent_name, name) # Ensure that basic validation for each spec part is done before it is added to spec cache if isinstance(spec, dict): self._error_cache.add( validate_required_attributes(fully_qualified_name, spec, ATTRIBUTE_NAME, ATTRIBUTE_TYPE)) if ATTRIBUTE_TYPE in spec and not Type.contains( spec[ATTRIBUTE_TYPE]): self._error_cache.add( InvalidTypeError(fully_qualified_name, spec, ATTRIBUTE_TYPE, InvalidTypeError.Reason.TYPE_NOT_DEFINED)) self._spec_cache[fully_qualified_name] = spec for key, val in spec.items(): if isinstance(val, list): for item in val: self.add_schema_spec(item, fully_qualified_name) self.add_schema_spec(val, fully_qualified_name) return spec[ATTRIBUTE_NAME]
def get_store(self, fully_qualified_name: str) -> Optional['Store']: """ Used to generate a store object from the given fully_qualified_name. :param fully_qualified_name: The fully qualified name of the store object needed. :return: An initialized store object """ if fully_qualified_name not in self._store_cache: schema = self.get_schema_object(fully_qualified_name) if not schema: return None if Type.is_store_type(schema.type): self._store_cache[fully_qualified_name] = TypeLoader.load_item( schema.type)(schema) else: self.add_errors( InvalidTypeError(fully_qualified_name, {}, ATTRIBUTE_TYPE, InvalidTypeError.Reason.INCORRECT_BASE, schema.type, InvalidTypeError.BaseTypes.STORE)) return self._store_cache.get(fully_qualified_name, None)
def test_is_type_equal_returns_true_when_actual_type_is_equal_string(): assert Type.is_type_equal("day", Type.DAY)
def test_is_type_equal_returns_false_when_actual_type_is_invalid_string(): assert Type.is_type_equal("invalid", Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_type(): assert Type.is_type_equal(Type.DAY, Type.HOUR) is False
def test_is_type_equal_returns_false_when_actual_type_is_unequal_string(): assert Type.is_type_equal("hour", Type.DAY) is False
def is_window_dtc(dtc_dict: Dict) -> bool: return Type.is_type_equal(dtc_dict.get('Type', ''), Type.BLURR_TRANSFORM_WINDOW)
def is_streaming_dtc(dtc_dict: Dict) -> bool: return Type.is_type_equal(dtc_dict.get('Type', ''), Type.BLURR_TRANSFORM_STREAMING)
def test_is_type_equal_returns_true_when_actual_type_is_equal_type(): assert Type.is_type_equal(Type.DAY, Type.DAY)