def _add_sqlalchemy_datasource(context, prompt_for_datasource_name=True):
msg_success_database = (
"\n<green>Great Expectations connected to your database!</green>"
)
if not _verify_sqlalchemy_dependent_modules():
return None
db_choices = [str(x) for x in list(range(1, 1 + len(SupportedDatabases)))]
selected_database = (
int(
click.prompt(
msg_prompt_choose_database,
type=click.Choice(db_choices),
show_choices=False,
)
)
- 1
) # don't show user a zero index list :)
selected_database = list(SupportedDatabases)[selected_database]
toolkit.send_usage_message(
data_context=context,
event="cli.new_ds_choice",
event_payload={"type": "sqlalchemy", "db": selected_database.name},
success=True,
)
datasource_name = "my_{}_db".format(selected_database.value.lower())
if selected_database == SupportedDatabases.OTHER:
datasource_name = "my_database"
if prompt_for_datasource_name:
datasource_name = click.prompt(
msg_prompt_datasource_name, default=datasource_name
)
credentials = {}
# Since we don't want to save the database credentials in the config file that will be
# committed in the repo, we will use our Variable Substitution feature to store the credentials
# in the credentials file (that will not be committed, since it is in the uncommitted directory)
# with the datasource's name as the variable name.
# The value of the datasource's "credentials" key in the config file (great_expectations.yml) will
# be ${datasource name}.
# Great Expectations will replace the ${datasource name} with the value from the credentials file in runtime.
while True:
cli_message(msg_db_config.format(datasource_name))
if selected_database == SupportedDatabases.MYSQL:
if not _verify_mysql_dependent_modules():
return None
credentials = _collect_mysql_credentials(default_credentials=credentials)
elif selected_database == SupportedDatabases.POSTGRES:
if not _verify_postgresql_dependent_modules():
return None
credentials = _collect_postgres_credentials(default_credentials=credentials)
elif selected_database == SupportedDatabases.REDSHIFT:
if not _verify_redshift_dependent_modules():
return None
credentials = _collect_redshift_credentials(default_credentials=credentials)
elif selected_database == SupportedDatabases.SNOWFLAKE:
if not _verify_snowflake_dependent_modules():
return None
credentials = _collect_snowflake_credentials(
default_credentials=credentials
)
elif selected_database == SupportedDatabases.BIGQUERY:
if not _verify_bigquery_dependent_modules():
return None
credentials = _collect_bigquery_credentials(default_credentials=credentials)
elif selected_database == SupportedDatabases.OTHER:
sqlalchemy_url = click.prompt(
"""What is the url/connection string for the sqlalchemy connection?
(reference: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls)
""",
show_default=False,
).strip()
credentials = {"url": sqlalchemy_url}
context.save_config_variable(datasource_name, credentials)
message = """
<red>Cannot connect to the database.</red>
- Please check your environment and the configuration you provided.
- Database Error: {0:s}"""
try:
cli_message(
"<cyan>Attempting to connect to your database. This may take a moment...</cyan>"
)
configuration = SqlAlchemyDatasource.build_configuration(
credentials="${" + datasource_name + "}"
)
configuration["class_name"] = "SqlAlchemyDatasource"
configuration["module_name"] = "great_expectations.datasource"
errors = DatasourceConfigSchema().validate(configuration)
if len(errors) != 0:
raise ge_exceptions.GreatExpectationsError(
"Invalid Datasource configuration: {:s}".format(errors)
)
cli_message(
"""
Great Expectations will now add a new Datasource '{0:s}' to your deployment, by adding this entry to your great_expectations.yml:
{1:s}
The credentials will be saved in uncommitted/config_variables.yml under the key '{0:s}'
""".format(
datasource_name,
textwrap.indent(
toolkit.yaml.dump({datasource_name: configuration}), " "
),
)
)
toolkit.confirm_proceed_or_exit()
context.add_datasource(name=datasource_name, **configuration)
cli_message(msg_success_database)
break
except ModuleNotFoundError as de:
cli_message(message.format(str(de)))
return None
except DatasourceInitializationError as de:
cli_message(message.format(str(de)))
if not click.confirm("Enter the credentials again?", default=True):
context.add_datasource(
datasource_name,
initialize=False,
module_name="great_expectations.datasource",
class_name="SqlAlchemyDatasource",
data_asset_type={"class_name": "SqlAlchemyDataset"},
credentials="${" + datasource_name + "}",
)
# TODO this message about continuing may not be accurate
cli_message(
"""
We saved datasource {:s} in {:s} and the credentials you entered in {:s}.
Since we could not connect to the database, you can complete troubleshooting in the configuration files documented in the how-to guides here:
<blue>https://docs.greatexpectations.io/en/latest/guides/how_to_guides/configuring_datasources.html?utm_source=cli&utm_medium=init&utm_campaign={:s}#{:s}</blue> .
After you connect to the datasource, run great_expectations init to continue.
""".format(
datasource_name,
DataContext.GE_YML,
context.get_config()["config_variables_file_path"],
rtd_url_ge_version,
selected_database.value.lower(),
)
)
return None
return datasource_name
def _add_sqlalchemy_datasource(context):
if not load_library("sqlalchemy"):
return None
db_choices = [str(x) for x in list(range(1, 1 + len(SupportedDatabases)))]
selected_database = int(
click.prompt(
msg_prompt_choose_database,
type=click.Choice(db_choices),
show_choices=False)) - 1 # don't show user a zero index list :)
selected_database = list(SupportedDatabases)[selected_database]
data_source_name = click.prompt(msg_prompt_datasource_name,
default="my_{}_db".format(
selected_database.value.lower()),
show_default=True)
credentials = {}
# Since we don't want to save the database credentials in the config file that will be
# committed in the repo, we will use our Variable Substitution feature to store the credentials
# in the credentials file (that will not be committed, since it is in the uncommitted directory)
# with the datasource's name as the variable name.
# The value of the datasource's "credentials" key in the config file (great_expectations.yml) will
# be ${datasource name}.
# GE will replace the ${datasource name} with the value from the credentials file in runtime.
while True:
cli_message(msg_db_config.format(data_source_name))
if selected_database == SupportedDatabases.MYSQL:
if not load_library("pymysql"):
return None
credentials = _collect_mysql_credentials(
default_credentials=credentials)
elif selected_database == SupportedDatabases.POSTGRES:
credentials = _collect_postgres_credentials(
default_credentials=credentials)
elif selected_database == SupportedDatabases.REDSHIFT:
if not load_library("psycopg2"):
return None
credentials = _collect_redshift_credentials(
default_credentials=credentials)
elif selected_database == SupportedDatabases.SNOWFLAKE:
if not load_library("snowflake",
install_instructions_string=
"pip install snowflake-sqlalchemy"):
return None
credentials = _collect_snowflake_credentials(
default_credentials=credentials)
elif selected_database == SupportedDatabases.OTHER:
sqlalchemy_url = click.prompt(
"""What is the url/connection string for the sqlalchemy connection?
(reference: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls)
""",
show_default=False)
credentials = {"url": sqlalchemy_url}
context.save_config_variable(data_source_name, credentials)
message = """
<red>Cannot connect to the database.</red>
- Please check your environment and the configuration you provided.
- Database Error: {0:s}"""
try:
configuration = SqlAlchemyDatasource.build_configuration(
credentials="${" + data_source_name + "}")
context.add_datasource(name=data_source_name,
class_name='SqlAlchemyDatasource',
**configuration)
break
except ModuleNotFoundError as de:
cli_message(message.format(str(de)))
return None
except DatasourceInitializationError as de:
cli_message(message.format(str(de)))
if not click.confirm("Enter the credentials again?".format(
str(de)),
default=True):
context.add_datasource(
data_source_name,
initialize=False,
module_name="great_expectations.datasource",
class_name="SqlAlchemyDatasource",
data_asset_type={"class_name": "SqlAlchemyDataset"},
credentials="${" + data_source_name + "}",
generators={"default": {
"class_name": "TableGenerator"
}})
cli_message("""
We saved datasource {0:s} in {1:s} and the credentials you entered in {2:s}.
Since we could not connect to the database, you can complete troubleshooting in the configuration files. Read here:
<blue>https://docs.greatexpectations.io/en/latest/tutorials/add-sqlalchemy-datasource.html?utm_source=cli&utm_medium=init&utm_campaign={3:s}#{4:s}</blue> .
After you connect to the datasource, run great_expectations profile to continue.
""".format(data_source_name, DataContext.GE_YML,
context.get_project_config().get("config_variables_file_path"),
rtd_url_ge_version, selected_database.value.lower()))
return None
return data_source_name
