class StorageServiceConfig(ServiceConfig): service_type: str = ConfigProperty(wysdom.SchemaConst('storage')) type: str = ConfigProperty(str) schema: str = ConfigProperty(str, default_function=_get_default_schema) drop_schema_if_exists: bool = ConfigProperty( bool, default_function=_get_default_drop_schema_if_exists)
class ComputeServiceConfig(ServiceConfig): service_type: str = ConfigProperty(wysdom.SchemaConst('compute')) storage_services: Dict[str, StorageServiceConfig] = ConfigProperty( wysdom.SchemaDict(StorageServiceConfig), default={}, persist_defaults=True) storage: StorageConfig = ConfigProperty(StorageConfig) schema: str = ConfigProperty(str, default_function=_get_default_schema) drop_schema_if_exists: bool = ConfigProperty( bool, default_function=_get_default_drop_schema_if_exists)
class SatelliteSQLPipeline(SatellitePipeline, register_as="sql"): type: str = wysdom.UserProperty(wysdom.SchemaConst('sql')) sql: str = wysdom.UserProperty(str, name="sql") load_dt: Optional[str] = wysdom.UserProperty(str, optional=True) deleted_ind: Optional[str] = wysdom.UserProperty(str, optional=True) dependencies: List[SatellitePipelineDependency] = wysdom.UserProperty( wysdom.SchemaArray(SatellitePipelineDependency), default=[]) @property def key_columns(self) -> Dict[str, str]: if self._key_columns: return self._key_columns else: return { hub_alias: hub_alias for hub_alias in self.satellite.parent.hubs.keys() }
class SatelliteSourcePipeline( SatellitePipeline, register_as="source" ): type: str = wysdom.UserProperty(wysdom.SchemaConst('source')) _source: str = wysdom.UserProperty(str, name="source") @property def key_columns(self) -> Dict[str, str]: if self._key_columns: return self._key_columns else: return { key_column: source_column for key_column, source_column in zip( self.satellite.parent.hubs.keys(), self.source.columns.keys() ) } @property def source(self) -> Source: # TODO: Refactor so this definitely returns Source, not VaultObject source_obj = self.project["source", self._source] assert isinstance(source_obj, Source) return source_obj @property def dependencies(self) -> List[SatellitePipelineDependency]: return [ SatellitePipelineDependency( {'name': self._source, 'type': 'source'}, json_dom_info=wysdom.dom.DOMInfo( document=wysdom.document(self), parent=self) ) ]
class SparkDeltaStorageConfig(StorageServiceConfig): type: str = wysdom.UserProperty(wysdom.SchemaConst('spark_delta'))
class LocalSparkConfig(ComputeServiceConfig): type: str = ConfigProperty(wysdom.SchemaConst('local_spark'))
class SimpleFileRegistryServiceConfig(RegistryServiceConfig): type: str = ConfigProperty(wysdom.SchemaConst('simple_file_registry')) storage_path: str = ConfigProperty(str)
class SQLAlchemyRegistryServiceConfig(RegistryServiceConfig): type: str = ConfigProperty(wysdom.SchemaConst('sqlalchemy_registry')) sqlalchemy_uri: str = ConfigProperty(str)
class RegistryServiceConfig(ServiceConfig): service_type: str = ConfigProperty(wysdom.SchemaConst('registry'))