class StorageServiceConfig(ServiceConfig):
service_type: str = ConfigProperty(wysdom.SchemaConst('storage'))
type: str = ConfigProperty(str)
schema: str = ConfigProperty(str, default_function=_get_default_schema)
drop_schema_if_exists: bool = ConfigProperty(
bool, default_function=_get_default_drop_schema_if_exists)
class ComputeServiceConfig(ServiceConfig):
service_type: str = ConfigProperty(wysdom.SchemaConst('compute'))
storage_services: Dict[str, StorageServiceConfig] = ConfigProperty(
wysdom.SchemaDict(StorageServiceConfig),
default={},
persist_defaults=True)
storage: StorageConfig = ConfigProperty(StorageConfig)
schema: str = ConfigProperty(str, default_function=_get_default_schema)
drop_schema_if_exists: bool = ConfigProperty(
bool, default_function=_get_default_drop_schema_if_exists)
class SatelliteSQLPipeline(SatellitePipeline, register_as="sql"):
type: str = wysdom.UserProperty(wysdom.SchemaConst('sql'))
sql: str = wysdom.UserProperty(str, name="sql")
load_dt: Optional[str] = wysdom.UserProperty(str, optional=True)
deleted_ind: Optional[str] = wysdom.UserProperty(str, optional=True)
dependencies: List[SatellitePipelineDependency] = wysdom.UserProperty(
wysdom.SchemaArray(SatellitePipelineDependency), default=[])
@property
def key_columns(self) -> Dict[str, str]:
if self._key_columns:
return self._key_columns
else:
return {
hub_alias: hub_alias
for hub_alias in self.satellite.parent.hubs.keys()
}
class SatelliteSourcePipeline(
SatellitePipeline,
register_as="source"
):
type: str = wysdom.UserProperty(wysdom.SchemaConst('source'))
_source: str = wysdom.UserProperty(str, name="source")
@property
def key_columns(self) -> Dict[str, str]:
if self._key_columns:
return self._key_columns
else:
return {
key_column: source_column
for key_column, source_column in zip(
self.satellite.parent.hubs.keys(),
self.source.columns.keys()
)
}
@property
def source(self) -> Source:
# TODO: Refactor so this definitely returns Source, not VaultObject
source_obj = self.project["source", self._source]
assert isinstance(source_obj, Source)
return source_obj
@property
def dependencies(self) -> List[SatellitePipelineDependency]:
return [
SatellitePipelineDependency(
{'name': self._source, 'type': 'source'},
json_dom_info=wysdom.dom.DOMInfo(
document=wysdom.document(self), parent=self)
)
]
class SparkDeltaStorageConfig(StorageServiceConfig):
type: str = wysdom.UserProperty(wysdom.SchemaConst('spark_delta'))
class LocalSparkConfig(ComputeServiceConfig):
type: str = ConfigProperty(wysdom.SchemaConst('local_spark'))
class SimpleFileRegistryServiceConfig(RegistryServiceConfig):
type: str = ConfigProperty(wysdom.SchemaConst('simple_file_registry'))
storage_path: str = ConfigProperty(str)
class SQLAlchemyRegistryServiceConfig(RegistryServiceConfig):
type: str = ConfigProperty(wysdom.SchemaConst('sqlalchemy_registry'))
sqlalchemy_uri: str = ConfigProperty(str)
class RegistryServiceConfig(ServiceConfig):
service_type: str = ConfigProperty(wysdom.SchemaConst('registry'))