def __init__(self, *args, **kwargs):
super(SparkDFDatasourceInMemoryBatchKwargs, self).__init__(*args, **kwargs)
try:
import pyspark
except ImportError:
raise InvalidBatchKwargsError(
"SparkDFDatasourceInMemoryBatchKwargs requires a valid pyspark installation, but pyspark import failed."
)
if not isinstance(self["dataset"], pyspark.sql.DataFrame):
raise InvalidBatchKwargsError("SparkDFDatasourceInMemoryBatchKwargs 'dataset' must be a spark DataFrame")
def __init__(self, *args, **kwargs):
super(SqlAlchemyDatasourceQueryBatchKwargs,
self).__init__(*args, **kwargs)
if "query" not in self:
raise InvalidBatchKwargsError(
"SqlAlchemyDatasourceQueryBatchKwargs requires a 'query' element"
)
def __init__(self, *args, **kwargs):
super(SqlAlchemyDatasourceTableBatchKwargs,
self).__init__(*args, **kwargs)
if "table" not in self:
raise InvalidBatchKwargsError(
"SqlAlchemyDatasourceTableBatchKwargs requires a 'table' element"
)
def __init__(self, *args, **kwargs):
super(PandasDatasourceInMemoryBatchKwargs, self).__init__(*args, **kwargs)
import pandas as pd
if not isinstance(self["dataset"], pd.DataFrame):
raise InvalidBatchKwargsError(
"PandasDatasourceInMemoryBatchKwargs 'dataset' must be a pandas DataFrame"
)
def _get_data_asset_config(self, data_asset_name):
if data_asset_name is None:
return
elif data_asset_name in self.assets:
return self.assets[data_asset_name]
raise InvalidBatchKwargsError(
"No asset definition for requested asset %s" % data_asset_name)
def _get_generator_asset_config(self, generator_asset):
if generator_asset is None:
return
elif generator_asset in self.assets:
return self.assets[generator_asset]
raise InvalidBatchKwargsError(
"No asset definition for requested asset %s" % generator_asset)
def __init__(self, *args, **kwargs):
super(InMemoryBatchKwargs, self).__init__(*args, **kwargs)
if "dataset" not in self:
raise InvalidBatchKwargsError(
"InMemoryBatchKwargs requires a 'dataset' element"
)
def __init__(self, *args, **kwargs):
super(S3BatchKwargs, self).__init__(*args, **kwargs)
if "s3" not in self:
raise InvalidBatchKwargsError("S3BatchKwargs requires a path element")
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if "path" not in self:
raise InvalidBatchKwargsError("PathBatchKwargs requires a path element")
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if "query" not in self:
raise InvalidBatchKwargsError(
"SparkDFDatasourceQueryBatchKwargs requires a 'query' element")
