def _add_pandas_datasource(datasource_name: str, dataset: AbstractDataSet,
ge_context: DataContext) -> str:
from great_expectations.datasource import PandasDatasource
path = str(dataset._filepath.parent)
if path.startswith("./"):
path = path[2:]
configuration = PandasDatasource.build_configuration(
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": os.path.join("..", path),
}
})
configuration["class_name"] = "PandasDatasource"
errors = DatasourceConfigSchema().validate(configuration)
if len(errors) != 0:
raise ge_exceptions.GreatExpectationsError(
"Invalid Datasource configuration: {0:s}".format(errors))
ge_context.add_datasource(name=datasource_name, **configuration)
return datasource_name
def _add_pandas_datasource(context):
path = click.prompt(
msg_prompt_filesys_enter_base_path,
# default='/data/',
type=click.Path(exists=True,
file_okay=False,
dir_okay=True,
readable=True),
show_default=True)
if path.startswith("./"):
path = path[2:]
if path.endswith("/"):
basenamepath = path[:-1]
else:
basenamepath = path
default_data_source_name = os.path.basename(basenamepath) + "__dir"
data_source_name = click.prompt(msg_prompt_datasource_name,
default=default_data_source_name,
show_default=True)
configuration = PandasDatasource.build_configuration(
base_directory=os.path.join("..", path))
context.add_datasource(name=data_source_name,
class_name='PandasDatasource',
**configuration)
return data_source_name
def _add_pandas_datasource(
context, passthrough_generator_only=True, prompt_for_datasource_name=True
):
toolkit.send_usage_message(
data_context=context,
event="cli.new_ds_choice",
event_payload={"type": "pandas"},
success=True,
)
if passthrough_generator_only:
datasource_name = "files_datasource"
configuration = PandasDatasource.build_configuration()
else:
path = click.prompt(
msg_prompt_filesys_enter_base_path,
type=click.Path(exists=True, file_okay=False),
)
if path.startswith("./"):
path = path[2:]
if path.endswith("/"):
basenamepath = path[:-1]
else:
basenamepath = path
datasource_name = os.path.basename(basenamepath) + "__dir"
if prompt_for_datasource_name:
datasource_name = click.prompt(
msg_prompt_datasource_name, default=datasource_name
)
configuration = PandasDatasource.build_configuration(
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": os.path.join("..", path),
}
}
)
configuration["class_name"] = "PandasDatasource"
configuration["module_name"] = "great_expectations.datasource"
errors = DatasourceConfigSchema().validate(configuration)
if len(errors) != 0:
raise ge_exceptions.GreatExpectationsError(
"Invalid Datasource configuration: {:s}".format(errors)
)
cli_message(
"""
Great Expectations will now add a new Datasource '{:s}' to your deployment, by adding this entry to your great_expectations.yml:
{:s}
""".format(
datasource_name,
textwrap.indent(toolkit.yaml.dump({datasource_name: configuration}), " "),
)
)
toolkit.confirm_proceed_or_exit(
continuation_message="Okay, exiting now. To learn more about adding datasources, run great_expectations "
"datasource --help or visit https://docs.greatexpectations.io/"
)
context.add_datasource(name=datasource_name, **configuration)
return datasource_name