def select_datasource(context: DataContext, datasource_name: str = None) -> Datasource:
"""Select a datasource interactively."""
# TODO consolidate all the myriad CLI tests into this
data_source = None
if datasource_name is None:
data_sources = sorted(context.list_datasources(), key=lambda x: x["name"])
if len(data_sources) == 0:
cli_message(
"<red>No datasources found in the context. To add a datasource, run `great_expectations datasource new`</red>"
)
elif len(data_sources) == 1:
datasource_name = data_sources[0]["name"]
else:
choices = "\n".join(
[
" {}. {}".format(i, data_source["name"])
for i, data_source in enumerate(data_sources, 1)
]
)
option_selection = click.prompt(
"Select a datasource" + "\n" + choices + "\n",
type=click.Choice(
[str(i) for i, data_source in enumerate(data_sources, 1)]
),
show_choices=False,
)
datasource_name = data_sources[int(option_selection) - 1]["name"]
if datasource_name is not None:
data_source = context.get_datasource(datasource_name)
return data_source
def delete_checkpoint(
context: DataContext,
checkpoint_name: str,
usage_event: str,
assume_yes: bool,
):
"""Delete a Checkpoint or raise helpful errors."""
validate_checkpoint(
context=context,
checkpoint_name=checkpoint_name,
usage_event=usage_event,
)
confirm_prompt: str = f"""\nAre you sure you want to delete the Checkpoint "{checkpoint_name}" (this action is irreversible)?"
"""
continuation_message: str = (
f'The Checkpoint "{checkpoint_name}" was not deleted. Exiting now.'
)
if not assume_yes:
confirm_proceed_or_exit(
confirm_prompt=confirm_prompt,
continuation_message=continuation_message,
data_context=context,
usage_stats_event=usage_event,
)
context.delete_checkpoint(name=checkpoint_name)
def load_data_context_with_error_handling(
directory: str, from_cli_upgrade_command: bool = False
) -> DataContext:
"""Return a DataContext with good error handling and exit codes."""
try:
context: DataContext = DataContext(context_root_dir=directory)
ge_config_version: int = context.get_config().config_version
if (
from_cli_upgrade_command
and int(ge_config_version) < CURRENT_GE_CONFIG_VERSION
):
directory = directory or context.root_directory
(
increment_version,
exception_occurred,
) = upgrade_project_one_version_increment(
context_root_dir=directory,
ge_config_version=ge_config_version,
continuation_message=EXIT_UPGRADE_CONTINUATION_MESSAGE,
from_cli_upgrade_command=from_cli_upgrade_command,
)
if not exception_occurred and increment_version:
context = DataContext(context_root_dir=directory)
return context
except ge_exceptions.UnsupportedConfigVersionError as err:
directory = directory or DataContext.find_context_root_dir()
ge_config_version = DataContext.get_ge_config_version(
context_root_dir=directory
)
upgrade_helper_class = (
GE_UPGRADE_HELPER_VERSION_MAP.get(int(ge_config_version))
if ge_config_version
else None
)
if upgrade_helper_class and ge_config_version < CURRENT_GE_CONFIG_VERSION:
upgrade_project(
context_root_dir=directory,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
else:
cli_message(f"<red>{err.message}</red>")
sys.exit(1)
except (
ge_exceptions.ConfigNotFoundError,
ge_exceptions.InvalidConfigError,
) as err:
cli_message(f"<red>{err.message}</red>")
sys.exit(1)
except ge_exceptions.PluginModuleNotFoundError as err:
cli_message(err.cli.v012_colored_message)
sys.exit(1)
except ge_exceptions.PluginClassNotFoundError as err:
cli_message(err.cli.v012_colored_message)
sys.exit(1)
except ge_exceptions.InvalidConfigurationYamlError as err:
cli_message(f"<red>{str(err)}</red>")
sys.exit(1)
def add_citation_with_batch_request(
data_context: DataContext,
expectation_suite: ExpectationSuite,
batch_request: Optional[Dict[str, Union[str, int, Dict[str, Any]]]] = None,
) -> None:
if (expectation_suite is not None and batch_request
and isinstance(batch_request, dict)
and BatchRequest(**batch_request)):
expectation_suite.add_citation(
comment="Created suite added via CLI",
batch_request=batch_request,
)
data_context.save_expectation_suite(
expectation_suite=expectation_suite)
def get_or_create_expectation_suite(
expectation_suite_name: str,
data_context: DataContext,
data_asset_name: Optional[str] = None,
usage_event: Optional[str] = None,
suppress_usage_message: Optional[bool] = False,
batch_request: Optional[
Union[str, Dict[str, Union[str, int, Dict[str, Any]]]]
] = None,
create_if_not_exist: Optional[bool] = True,
) -> ExpectationSuite:
if expectation_suite_name is None:
default_expectation_suite_name: str = get_default_expectation_suite_name(
data_asset_name=data_asset_name,
batch_request=batch_request,
)
while True:
expectation_suite_name = click.prompt(
"\nName the new Expectation Suite",
default=default_expectation_suite_name,
)
if (
expectation_suite_name
not in data_context.list_expectation_suite_names()
):
break
tell_user_suite_exists(
data_context=data_context,
expectation_suite_name=expectation_suite_name,
usage_event=usage_event,
suppress_usage_message=suppress_usage_message,
)
elif expectation_suite_name in data_context.list_expectation_suite_names():
tell_user_suite_exists(
data_context=data_context,
expectation_suite_name=expectation_suite_name,
usage_event=usage_event,
suppress_usage_message=suppress_usage_message,
)
suite: ExpectationSuite = load_expectation_suite(
data_context=data_context,
expectation_suite_name=expectation_suite_name,
usage_event=usage_event,
suppress_usage_message=suppress_usage_message,
create_if_not_exist=create_if_not_exist,
)
return suite
def upgrade_project_strictly_multiple_versions_increment(
directory: str,
ge_config_version: float,
from_cli_upgrade_command: bool = False) -> Optional[DataContext]:
upgrade_helper_class = (GE_UPGRADE_HELPER_VERSION_MAP.get(
int(ge_config_version)) if ge_config_version else None)
context: Optional[DataContext]
if upgrade_helper_class and int(
ge_config_version) < CURRENT_GE_CONFIG_VERSION:
upgrade_project(
context_root_dir=directory,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
context = DataContext(context_root_dir=directory)
# noinspection PyBroadException
try:
send_usage_message(
data_context=context,
event="cli.project.upgrade.end",
success=True,
)
except Exception:
# Don't fail for usage stats
pass
else:
context = None
return context
def run_checkpoint(
context: DataContext,
checkpoint_name: str,
usage_event: str,
) -> CheckpointResult:
"""Run a Checkpoint or raise helpful errors."""
failure_message: str = "Exception occurred while running Checkpoint."
validate_checkpoint(
context=context,
checkpoint_name=checkpoint_name,
usage_event=usage_event,
failure_message=failure_message,
)
try:
result: CheckpointResult = context.run_checkpoint(
checkpoint_name=checkpoint_name
)
return result
except ge_exceptions.CheckpointError as e:
cli_message(string=failure_message)
exit_with_failure_message_and_stats(
data_context=context,
usage_event=usage_event,
message=f"<red>{e}.</red>",
)
def load_checkpoint(
context: DataContext,
checkpoint_name: str,
usage_event: str,
) -> Union[Checkpoint, LegacyCheckpoint]:
"""Load a checkpoint or raise helpful errors."""
try:
checkpoint: Union[Checkpoint, LegacyCheckpoint] = context.get_checkpoint(
name=checkpoint_name
)
return checkpoint
except (
ge_exceptions.CheckpointNotFoundError,
ge_exceptions.InvalidCheckpointConfigError,
):
exit_with_failure_message_and_stats(
context,
usage_event,
f"""\
<red>Could not find checkpoint `{checkpoint_name}`.</red> Try running:
- `<green>great_expectations checkpoint list</green>` to verify your checkpoint exists
- `<green>great_expectations checkpoint new</green>` to configure a new checkpoint""",
)
except ge_exceptions.CheckpointError as e:
exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>")
def select_datasource(context: DataContext,
datasource_name: str = None) -> BaseDatasource:
"""Select a datasource interactively."""
# TODO consolidate all the myriad CLI tests into this
data_source: Optional[BaseDatasource] = None
if datasource_name is None:
data_sources: List[BaseDatasource] = cast(
List[BaseDatasource],
list(sorted(context.datasources.values(), key=lambda x: x.name), ),
)
if len(data_sources) == 0:
cli_message(
string=
"<red>No datasources found in the context. To add a datasource, run `great_expectations datasource new`</red>"
)
elif len(data_sources) == 1:
datasource_name = data_sources[0].name
else:
choices: str = "\n".join([
f" {i}. {data_source.name}"
for i, data_source in enumerate(data_sources, 1)
])
option_selection: str = click.prompt(
"Select a datasource" + "\n" + choices + "\n",
type=click.Choice(
[str(i) for i, data_source in enumerate(data_sources, 1)]),
show_choices=False,
)
datasource_name = data_sources[int(option_selection) - 1].name
if datasource_name is not None:
data_source = context.get_datasource(datasource_name=datasource_name)
return data_source
def create_empty_suite(context: DataContext, expectation_suite_name: str,
batch_kwargs) -> None:
cli_message("""
Great Expectations will create a new Expectation Suite '{:s}' and store it here:
{:s}
""".format(
expectation_suite_name,
context.stores[
context.expectations_store_name].store_backend.get_url_for_key(
ExpectationSuiteIdentifier(
expectation_suite_name=expectation_suite_name).to_tuple()),
))
suite = context.create_expectation_suite(expectation_suite_name)
suite.add_citation(comment="New suite added via CLI",
batch_kwargs=batch_kwargs)
context.save_expectation_suite(suite, expectation_suite_name)
def load_batch(
context: DataContext,
suite: Union[str, ExpectationSuite],
batch_kwargs: Union[dict, BatchKwargs],
) -> Union[Batch, DataAsset]:
batch: Union[Batch, DataAsset] = context.get_batch(batch_kwargs, suite)
assert isinstance(batch, DataAsset) or isinstance(
batch, Batch), "Batch failed to load. Please check your batch_kwargs"
return batch
def load_expectation_suite(
data_context: DataContext,
expectation_suite_name: str,
usage_event: str,
suppress_usage_message: Optional[bool] = False,
create_if_not_exist: Optional[bool] = True,
) -> Optional[ExpectationSuite]:
"""
Load an expectation suite from a given context.
Handles a suite name with or without `.json`
:param data_context:
:param expectation_suite_name:
:param usage_event:
:param suppress_usage_message:
:param create_if_not_exist:
"""
if expectation_suite_name.endswith(".json"):
expectation_suite_name = expectation_suite_name[:-5]
suite: Optional[ExpectationSuite]
try:
suite = data_context.get_expectation_suite(
expectation_suite_name=expectation_suite_name
)
return suite
except ge_exceptions.DataContextError:
if create_if_not_exist:
suite = data_context.create_expectation_suite(
expectation_suite_name=expectation_suite_name
)
return suite
else:
suite = None
exit_with_failure_message_and_stats(
data_context=data_context,
usage_event=usage_event,
suppress_usage_message=suppress_usage_message,
message=f"<red>Could not find a suite named `{expectation_suite_name}`.</red> Please check "
"the name by running `great_expectations suite list` and try again.",
)
return suite
def get_validator(
context: DataContext,
batch_request: Union[dict, BatchRequest],
suite: Union[str, ExpectationSuite],
) -> Validator:
assert isinstance(
suite,
(str, ExpectationSuite
)), "Invalid suite type (must be ExpectationSuite) or a string."
if isinstance(batch_request, dict):
batch_request = BatchRequest(**batch_request)
validator: Validator
if isinstance(suite, str):
validator = context.get_validator(batch_request=batch_request,
expectation_suite_name=suite)
else:
validator = context.get_validator(batch_request=batch_request,
expectation_suite=suite)
return validator
def empty_data_context_with_config_variables(monkeypatch, empty_data_context):
monkeypatch.setenv("FOO", "BAR")
monkeypatch.setenv("REPLACE_ME_ESCAPED_ENV", "ive_been_$--replaced")
root_dir = empty_data_context.root_directory
ge_config_path = file_relative_path(
__file__,
"../test_fixtures/great_expectations_basic_with_variables.yml",
)
shutil.copy(ge_config_path, os.path.join(root_dir,
"great_expectations.yml"))
config_variables_path = file_relative_path(
__file__,
"../test_fixtures/config_variables.yml",
)
shutil.copy(config_variables_path, os.path.join(root_dir, "uncommitted"))
return DataContext(context_root_dir=root_dir)
def load_expectation_suite(
# TODO consolidate all the myriad CLI tests into this
context: DataContext,
suite_name: str,
usage_event: str,
) -> ExpectationSuite:
"""
Load an expectation suite from a given context.
Handles a suite name with or without `.json`
:param usage_event:
"""
if suite_name.endswith(".json"):
suite_name = suite_name[:-5]
try:
suite = context.get_expectation_suite(suite_name)
return suite
except ge_exceptions.DataContextError as e:
exit_with_failure_message_and_stats(
context,
usage_event,
f"<red>Could not find a suite named `{suite_name}`.</red> Please check "
"the name by running `great_expectations suite list` and try again.",
)
def test_evaluation_parameter_store_methods(
data_context_parameterized_expectation_suite: DataContext, ):
run_id = RunIdentifier(run_name="20191125T000000.000000Z")
source_patient_data_results = ExpectationSuiteValidationResult(
meta={
"expectation_suite_name": "source_patient_data.default",
"run_id": run_id,
},
results=[
ExpectationValidationResult(
expectation_config=ExpectationConfiguration(
expectation_type="expect_table_row_count_to_equal",
kwargs={
"value": 1024,
},
),
success=True,
exception_info={
"exception_message": None,
"exception_traceback": None,
"raised_exception": False,
},
result={
"observed_value": 1024,
"element_count": 1024,
"missing_percent": 0.0,
"missing_count": 0,
},
)
],
success=True,
)
data_context_parameterized_expectation_suite.store_evaluation_parameters(
source_patient_data_results)
bound_parameters = data_context_parameterized_expectation_suite.evaluation_parameter_store.get_bind_params(
run_id)
assert bound_parameters == {
"urn:great_expectations:validations:source_patient_data.default:expect_table_row_count_to_equal.result"
".observed_value":
1024
}
source_diabetes_data_results = ExpectationSuiteValidationResult(
meta={
"expectation_suite_name": "source_diabetes_data.default",
"run_id": run_id,
},
results=[
ExpectationValidationResult(
expectation_config=ExpectationConfiguration(
expectation_type=
"expect_column_unique_value_count_to_be_between",
kwargs={
"column": "patient_nbr",
"min": 2048,
"max": 2048
},
),
success=True,
exception_info={
"exception_message": None,
"exception_traceback": None,
"raised_exception": False,
},
result={
"observed_value": 2048,
"element_count": 5000,
"missing_percent": 0.0,
"missing_count": 0,
},
)
],
success=True,
)
data_context_parameterized_expectation_suite.store_evaluation_parameters(
source_diabetes_data_results)
bound_parameters = data_context_parameterized_expectation_suite.evaluation_parameter_store.get_bind_params(
run_id)
assert bound_parameters == {
"urn:great_expectations:validations:source_patient_data.default:expect_table_row_count_to_equal.result"
".observed_value":
1024,
"urn:great_expectations:validations:source_diabetes_data.default"
":expect_column_unique_value_count_to_be_between.result.observed_value:column=patient_nbr":
2048,
}
import numpy as np
from great_expectations.core.batch import BatchRequest
from great_expectations.data_context.data_context import DataContext
from great_expectations.datasource.data_connector.batch_filter import (
BatchFilter,
build_batch_filter,
)
from great_expectations.validator.validation_graph import MetricConfiguration
context = DataContext()
suite = context.get_expectation_suite("yellow_trip_data_validations")
# This BatchRequest will retrieve all twelve batches from 2019
multi_batch_request = BatchRequest(
datasource_name="taxi_pandas",
data_connector_name="monthly",
data_asset_name="my_reports",
data_connector_query={"batch_filter_parameters": {"year": "2019"}},
)
# Instantiate the Validator
validator_multi_batch = context.get_validator(
batch_request=multi_batch_request, expectation_suite=suite
)
# The active batch should be December, as this should be the last one loaded. Confirming here.
assert validator_multi_batch.active_batch_definition.batch_identifiers["month"] == "12"
# Get the list of all batches contained by the Validator for use in the BatchFilter
total_batch_definition_list: list = [
def upgrade_project(
context_root_dir, ge_config_version, from_cli_upgrade_command=False
):
if from_cli_upgrade_command:
message = (
f"<red>\nYour project appears to have an out-of-date config version ({ge_config_version}) - "
f"the version "
f"number must be at least {CURRENT_GE_CONFIG_VERSION}.</red>"
)
else:
message = (
f"<red>\nYour project appears to have an out-of-date config version ({ge_config_version}) - "
f"the version "
f"number must be at least {CURRENT_GE_CONFIG_VERSION}.\nIn order to proceed, "
f"your project must be upgraded.</red>"
)
cli_message(string=message)
upgrade_prompt = (
"\nWould you like to run the Upgrade Helper to bring your project up-to-date?"
)
# This loading of DataContext is optional and just to track if someone exits here
try:
data_context = DataContext(context_root_dir)
except Exception:
# Do not raise error for usage stats
data_context = None
confirm_proceed_or_exit(
confirm_prompt=upgrade_prompt,
continuation_message=EXIT_UPGRADE_CONTINUATION_MESSAGE,
data_context=data_context,
usage_stats_event="cli.project.upgrade.end",
)
cli_message(string=SECTION_SEPARATOR)
# use loop in case multiple upgrades need to take place
while ge_config_version < CURRENT_GE_CONFIG_VERSION:
increment_version, exception_occurred = upgrade_project_one_version_increment(
context_root_dir=context_root_dir,
ge_config_version=ge_config_version,
continuation_message=EXIT_UPGRADE_CONTINUATION_MESSAGE,
from_cli_upgrade_command=from_cli_upgrade_command,
)
if exception_occurred or not increment_version:
break
ge_config_version += 1
cli_message(string=SECTION_SEPARATOR)
upgrade_success_message = "<green>Upgrade complete. Exiting...</green>\n"
upgrade_incomplete_message = f"""\
<red>The Upgrade Helper was unable to perform a complete project upgrade. Next steps:</red>
- Please perform any manual steps outlined in the Upgrade Overview and/or Upgrade Report above
- When complete, increment the config_version key in your <cyan>great_expectations.yml</cyan> to <cyan>{
ge_config_version + 1}</cyan>\n
To learn more about the upgrade process, visit \
<cyan>https://docs.greatexpectations.io/en/latest/how_to_guides/migrating_versions.html</cyan>
"""
if ge_config_version < CURRENT_GE_CONFIG_VERSION:
cli_message(string=upgrade_incomplete_message)
# noinspection PyBroadException
try:
context: DataContext = DataContext(context_root_dir=context_root_dir)
send_usage_message(
data_context=context, event="cli.project.upgrade.end", success=False
)
except Exception:
# Do not raise error for usage stats
pass
else:
cli_message(upgrade_success_message)
try:
context: DataContext = DataContext(context_root_dir)
send_usage_message(
data_context=context, event="cli.project.upgrade.end", success=True
)
except Exception:
# Do not raise error for usage stats
pass
sys.exit(0)
def load_data_context_with_error_handling(
directory: str,
from_cli_upgrade_command: bool = False) -> Optional[DataContext]:
"""Return a DataContext with good error handling and exit codes."""
context: Optional[DataContext]
ge_config_version: float
try:
directory = directory or DataContext.find_context_root_dir()
context = DataContext(context_root_dir=directory)
ge_config_version = context.get_config().config_version
if from_cli_upgrade_command:
if ge_config_version < CURRENT_GE_CONFIG_VERSION:
context = upgrade_project_one_or_multiple_versions_increment(
directory=directory,
context=context,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
elif ge_config_version > CURRENT_GE_CONFIG_VERSION:
raise ge_exceptions.UnsupportedConfigVersionError(
f"""Invalid config version ({ge_config_version}).\n The maximum valid version is \
{CURRENT_GE_CONFIG_VERSION}.
""")
else:
context = upgrade_project_zero_versions_increment(
directory=directory,
context=context,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
return context
except ge_exceptions.UnsupportedConfigVersionError as err:
directory = directory or DataContext.find_context_root_dir()
ge_config_version = DataContext.get_ge_config_version(
context_root_dir=directory)
context = upgrade_project_strictly_multiple_versions_increment(
directory=directory,
ge_config_version=ge_config_version,
from_cli_upgrade_command=from_cli_upgrade_command,
)
if context:
return context
else:
cli_message(string=f"<red>{err.message}</red>")
sys.exit(1)
except (
ge_exceptions.ConfigNotFoundError,
ge_exceptions.InvalidConfigError,
) as err:
cli_message(string=f"<red>{err.message}</red>")
sys.exit(1)
except ge_exceptions.PluginModuleNotFoundError as err:
cli_message(string=err.cli_colored_message)
sys.exit(1)
except ge_exceptions.PluginClassNotFoundError as err:
cli_message(string=err.cli_colored_message)
sys.exit(1)
except ge_exceptions.InvalidConfigurationYamlError as err:
cli_message(string=f"<red>{str(err)}</red>")
sys.exit(1)
from typing import List
import numpy as np
from great_expectations.core.batch import BatchRequest
from great_expectations.data_context.data_context import DataContext
from great_expectations.datasource.data_connector.batch_filter import (
BatchFilter,
build_batch_filter,
)
from great_expectations.validator.validation_graph import MetricConfiguration
from great_expectations.validator.validator import Validator
context = DataContext()
suite = context.get_expectation_suite("yellow_trip_data_validations")
# Create three BatchRequests for Jan, Feb, and March 2019 data and instantiate a Validator with all three BatchRequests
jan_batch_request: BatchRequest = BatchRequest(
datasource_name="taxi_pandas",
data_connector_name="monthly",
data_asset_name="my_reports",
data_connector_query={
"batch_filter_parameters": {
"month": "01",
"year": "2019"
}
},
)
feb_batch_request: BatchRequest = BatchRequest(
datasource_name="taxi_pandas",
def upgrade_project_zero_versions_increment(
directory: str,
context: DataContext,
ge_config_version: float,
from_cli_upgrade_command: bool = False,
) -> Optional[DataContext]:
upgrade_helper_class = (GE_UPGRADE_HELPER_VERSION_MAP.get(
int(ge_config_version)) if ge_config_version else None)
if upgrade_helper_class:
upgrade_helper = upgrade_helper_class(context_root_dir=directory,
update_version=False)
else:
error_message: str = (
f"The upgrade utility for version {ge_config_version} could not be found."
)
cli_message(string=f"<red>{error_message}</red>")
sys.exit(1)
manual_steps_required = upgrade_helper.manual_steps_required()
if manual_steps_required:
# noinspection PyBroadException
try:
send_usage_message(
data_context=context,
event="cli.project.upgrade.begin",
success=True,
)
except Exception:
# Don't fail for usage stats
pass
(
increment_version,
exception_occurred,
) = upgrade_project_up_to_one_version_increment(
context_root_dir=directory,
ge_config_version=ge_config_version,
continuation_message=EXIT_UPGRADE_CONTINUATION_MESSAGE,
update_version=False,
from_cli_upgrade_command=from_cli_upgrade_command,
)
if exception_occurred or increment_version:
context = None
else:
if manual_steps_required:
upgrade_message = "Your project requires manual upgrade steps in order to be up-to-date.\n"
cli_message(f"<yellow>{upgrade_message}</yellow>")
else:
upgrade_message = (
"Your project is up-to-date - no further upgrade is necessary.\n"
)
cli_message(f"<green>{upgrade_message}</green>")
context = DataContext(context_root_dir=directory)
# noinspection PyBroadException
try:
send_usage_message(
data_context=context,
event="cli.project.upgrade.end",
success=True,
)
except Exception:
# Don't fail for usage stats
pass
return context
from great_expectations.core.batch import BatchRequest
from great_expectations.data_context.data_context import DataContext
from great_expectations.validator.validation_graph import MetricConfiguration
context = DataContext()
suite = context.get_expectation_suite("yellow_trip_data_validations")
# Get February BatchRequest and Validator
batch_request_february = BatchRequest(
datasource_name="taxi_pandas",
data_connector_name="monthly",
data_asset_name="my_reports",
data_connector_query={"index": -2},
)
validator_february = context.get_validator(
batch_request=batch_request_february, expectation_suite=suite)
# Get the table row count for February
february_table_row_count = validator_february.get_metric(
MetricConfiguration("table.row_count", metric_domain_kwargs={}))
# Get March BatchRequest and Validator
batch_request_march = BatchRequest(
datasource_name="taxi_pandas",
data_connector_name="monthly",
data_asset_name="my_reports",
data_connector_query={"index": -1},
)
validator_march = context.get_validator(batch_request=batch_request_march,
expectation_suite=suite)
def upgrade_project_one_version_increment(
context_root_dir: str,
ge_config_version: float,
continuation_message: str,
from_cli_upgrade_command: bool = False,
) -> [bool, bool]: # Returns increment_version, exception_occurred
upgrade_helper_class = GE_UPGRADE_HELPER_VERSION_MAP.get(int(ge_config_version))
if not upgrade_helper_class:
return False, False
target_ge_config_version = int(ge_config_version) + 1
# set version temporarily to CURRENT_GE_CONFIG_VERSION to get functional DataContext
DataContext.set_ge_config_version(
config_version=CURRENT_GE_CONFIG_VERSION,
context_root_dir=context_root_dir,
)
upgrade_helper = upgrade_helper_class(context_root_dir=context_root_dir)
upgrade_overview, confirmation_required = upgrade_helper.get_upgrade_overview()
if confirmation_required or from_cli_upgrade_command:
upgrade_confirmed = confirm_proceed_or_exit(
confirm_prompt=upgrade_overview,
continuation_message=continuation_message,
exit_on_no=False,
)
else:
upgrade_confirmed = True
if upgrade_confirmed:
cli_message("\nUpgrading project...")
cli_message(SECTION_SEPARATOR)
# run upgrade and get report of what was done, if version number should be incremented
(
upgrade_report,
increment_version,
exception_occurred,
) = upgrade_helper.upgrade_project()
# display report to user
cli_message(upgrade_report)
if exception_occurred:
# restore version number to current number
DataContext.set_ge_config_version(
ge_config_version, context_root_dir, validate_config_version=False
)
# display report to user
return False, True
# set config version to target version
if increment_version:
DataContext.set_ge_config_version(
target_ge_config_version,
context_root_dir,
validate_config_version=False,
)
return True, False
# restore version number to current number
DataContext.set_ge_config_version(
ge_config_version, context_root_dir, validate_config_version=False
)
return False, False
# restore version number to current number
DataContext.set_ge_config_version(
ge_config_version, context_root_dir, validate_config_version=False
)
cli_message(continuation_message)
sys.exit(0)
from typing import List
import numpy as np
from great_expectations.core.batch import Batch, BatchRequest
from great_expectations.data_context.data_context import DataContext
from great_expectations.datasource.data_connector.batch_filter import (
BatchFilter,
build_batch_filter,
)
from great_expectations.validator.metric_configuration import MetricConfiguration
from great_expectations.validator.validator import Validator
context = DataContext()
suite = context.get_expectation_suite("yellow_tripdata_validations")
# Create a BatchRequest and instantiate a Validator with only the January 2019 data
jan_batch_request: BatchRequest = BatchRequest(
datasource_name="taxi_pandas",
data_connector_name="monthly",
data_asset_name="my_reports",
data_connector_query={
"batch_filter_parameters": {
"month": "01",
"year": "2019"
}
},
)
validator: Validator = context.get_validator(batch_request=jan_batch_request,
expectation_suite=suite)