def suite_delete(suite, directory): """ Delete an expectation suite from the expectation store. """ usage_event = "cli.suite.delete" context = toolkit.load_data_context_with_error_handling(directory) suite_names = context.list_expectation_suite_names() if not suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, "</red>No expectation suites found in the project.</red>", ) if suite not in suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, f"No expectation suite named {suite} found.") context.delete_expectation_suite(suite) cli_message(f"Deleted the expectation suite named: {suite}") send_usage_message(data_context=context, event=usage_event, success=True)
def clean_data_docs(directory, site_name=None, all=None): """Delete data docs""" context = toolkit.load_data_context_with_error_handling(directory) failed = True if site_name is None and all is None: cli_message("<red>{}</red>".format( "Please specify --all to remove all sites or specify a specific site using " "--site_name")) sys.exit(1) context.clean_data_docs(site_name=site_name) failed = False if not failed and context is not None: send_usage_message(data_context=context, event="cli.docs.clean", success=True) cli_message("<green>{}</green>".format("Cleaned data docs")) if failed and context is not None: send_usage_message(data_context=context, event="cli.docs.clean", success=False)
def store_list(directory): """List known Stores.""" context = toolkit.load_data_context_with_error_handling(directory) try: stores = context.list_stores() if len(stores) == 0: cli_message("No Stores found") send_usage_message(data_context=context, event="cli.store.list", success=True) return elif len(stores) == 1: list_intro_string = "1 Store found:" else: list_intro_string = "{} Stores found:".format(len(stores)) cli_message(list_intro_string) for store in stores: cli_message("") cli_message_dict(store) send_usage_message(data_context=context, event="cli.store.list", success=True) except Exception as e: send_usage_message(data_context=context, event="cli.store.list", success=False) raise e
def _suite_scaffold(suite: str, directory: str, jupyter: bool) -> None: usage_event = "cli.suite.scaffold" suite_name = suite context = load_data_context_with_error_handling(directory) notebook_filename = f"scaffold_{suite_name}.ipynb" notebook_path = _get_notebook_path(context, notebook_filename) if suite_name in context.list_expectation_suite_names(): toolkit.tell_user_suite_exists(suite_name) if os.path.isfile(notebook_path): cli_message( f" - If you wish to adjust your scaffolding, you can open this notebook with jupyter: `{notebook_path}` <red>(Please note that if you run that notebook, you will overwrite your existing suite.)</red>" ) send_usage_message(data_context=context, event=usage_event, success=False) sys.exit(1) datasource = toolkit.select_datasource(context) if datasource is None: send_usage_message(data_context=context, event=usage_event, success=False) sys.exit(1) _suite = context.create_expectation_suite(suite_name) _, _, _, batch_kwargs = get_batch_kwargs(context, datasource_name=datasource.name) renderer = SuiteScaffoldNotebookRenderer(context, _suite, batch_kwargs) renderer.render_to_disk(notebook_path) if jupyter: toolkit.launch_jupyter_notebook(notebook_path) else: cli_message( f"To continue scaffolding this suite, run `jupyter notebook {notebook_path}`" ) send_usage_message(data_context=context, event=usage_event, success=True)
def suite_list(directory): """Lists available Expectation Suites.""" context = load_data_context_with_error_handling(directory) try: suite_names = [ " - <cyan>{}</cyan>".format(suite_name) for suite_name in context.list_expectation_suite_names() ] if len(suite_names) == 0: cli_message("No Expectation Suites found") send_usage_message( data_context=context, event="cli.suite.list", success=True ) return elif len(suite_names) == 1: list_intro_string = "1 Expectation Suite found:" else: list_intro_string = "{} Expectation Suites found:".format(len(suite_names)) cli_message_list(suite_names, list_intro_string) send_usage_message(data_context=context, event="cli.suite.list", success=True) except Exception as e: send_usage_message(data_context=context, event="cli.suite.list", success=False) raise e
def checkpoint_new(checkpoint, suite, directory, datasource, legacy): """Create a new checkpoint for easy deployments. (Experimental)""" if legacy: suite_name = suite usage_event = "cli.checkpoint.new" context = toolkit.load_data_context_with_error_handling(directory) _verify_checkpoint_does_not_exist(context, checkpoint, usage_event) suite: ExpectationSuite = toolkit.load_expectation_suite( context, suite_name, usage_event) datasource = toolkit.select_datasource(context, datasource_name=datasource) if datasource is None: send_usage_message(context, usage_event, success=False) sys.exit(1) _, _, _, batch_kwargs = toolkit.get_batch_kwargs( context, datasource.name) _ = context.add_checkpoint( name=checkpoint, **{ "class_name": "LegacyCheckpoint", "validation_operator_name": "action_list_operator", "batches": [{ "batch_kwargs": dict(batch_kwargs), "expectation_suite_names": [suite.expectation_suite_name], }], }, ) cli_message( f"""<green>A checkpoint named `{checkpoint}` was added to your project!</green> - To run this checkpoint run `great_expectations checkpoint run {checkpoint}`""" ) send_usage_message(context, usage_event, success=True) # TODO: <Rob>Rob</Rob> Add flow for new style checkpoints else: pass
def docs_list(directory): """List known Data Docs Sites.""" context = toolkit.load_data_context_with_error_handling(directory) docs_sites_url_dicts = context.get_docs_sites_urls() docs_sites_strings = [ " - <cyan>{}</cyan>: {}".format( docs_site_dict["site_name"], docs_site_dict.get("site_url") or f"site configured but does not exist. Run the following command to build site: great_expectations " f'docs build --site-name {docs_site_dict["site_name"]}', ) for docs_site_dict in docs_sites_url_dicts ] if len(docs_sites_strings) == 0: cli_message("No Data Docs sites found") else: list_intro_string = _build_intro_string(docs_sites_strings) cli_message_list(docs_sites_strings, list_intro_string) send_usage_message(data_context=context, event="cli.docs.list", success=True)
def _tap_new(suite, tap_filename, directory, usage_event, datasource=None): context = load_data_context_with_error_handling(directory) try: _validate_tap_filename(tap_filename) context_directory = context.root_directory datasource = _get_datasource(context, datasource) suite = load_expectation_suite(context, suite) _, _, _, batch_kwargs = get_batch_kwargs(context, datasource.name) tap_filename = _write_tap_file_to_disk(batch_kwargs, context_directory, suite, tap_filename) cli_message(f"""\ <green>A new tap has been generated!</green> To run this tap, run: <green>python {tap_filename}</green> You can edit this script or place this code snippet in your pipeline.""") send_usage_message(data_context=context, event=usage_event, success=True) except Exception as e: send_usage_message(data_context=context, event=usage_event, success=False) raise e
def suite_list(directory): """Lists available Expectation Suites.""" try: context = DataContext(directory) except ge_exceptions.ConfigNotFoundError as err: cli_message("<red>{}</red>".format(err.message)) return try: suite_names = [ " - <cyan>{}</cyan>".format(suite_name) for suite_name in context.list_expectation_suite_names() ] if len(suite_names) == 0: cli_message("No Expectation Suites found") send_usage_message( data_context=context, event="cli.suite.list", success=True ) return if len(suite_names) == 1: list_intro_string = "1 Expectation Suite found:" if len(suite_names) > 1: list_intro_string = "{} Expectation Suites found:".format(len(suite_names)) cli_message_list(suite_names, list_intro_string) send_usage_message( data_context=context, event="cli.suite.list", success=True ) except Exception as e: send_usage_message( data_context=context, event="cli.suite.list", success=False ) raise e
def docs_list(directory): """List known Data Docs Sites.""" context = None try: failed = True context = DataContext(directory) docs_sites_url_dicts = context.get_docs_sites_urls() docs_sites_strings = [ " - <cyan>{}</cyan>: {}".format(docs_site_dict["site_name"], docs_site_dict["site_url"])\ for docs_site_dict in docs_sites_url_dicts ] if len(docs_sites_strings) == 0: cli_message("No Data Docs sites found") failed = False send_usage_message(data_context=context, event="cli.docs.list", success=True) return if len(docs_sites_strings) == 1: list_intro_string = "1 Data Docs site found:" if len(docs_sites_strings) > 1: list_intro_string = "{} Data Docs sites found:".format( len(docs_sites_strings)) cli_message_list(docs_sites_strings, list_intro_string) failed = False send_usage_message(data_context=context, event="cli.docs.list", success=True) except ge_exceptions.ConfigNotFoundError as err: cli_message("<red>{}</red>".format(err.message)) finally: if failed and context is not None: send_usage_message(data_context=context, event="cli.docs.list", success=False)
def store_list(directory): """List known Stores.""" try: context = DataContext(directory) except ge_exceptions.ConfigNotFoundError as err: cli_message("<red>{}</red>".format(err.message)) return try: stores = context.list_stores() if len(stores) == 0: cli_message("No Stores found") send_usage_message(data_context=context, event="cli.store.list", success=True) return if len(stores) == 1: list_intro_string = "1 Store found:" if len(stores) > 1: list_intro_string = "{} Stores found:".format(len(stores)) cli_message(list_intro_string) for store in stores: cli_message("") cli_message_dict(store) send_usage_message(data_context=context, event="cli.store.list", success=True) except Exception as e: send_usage_message(data_context=context, event="cli.store.list", success=False) raise e
def checkpoint_run(checkpoint, directory): """Run a checkpoint. (Experimental)""" usage_event = "cli.checkpoint.run" context = toolkit.load_data_context_with_error_handling( directory=directory, from_cli_upgrade_command=False) ge_config_version = context.get_config().config_version if ge_config_version >= 3: cli_message( f"""<red>The `checkpoint run` CLI command is not yet implemented for GE config versions >= 3.</red>""" ) send_usage_message(context, usage_event, success=False) sys.exit(1) checkpoint: Checkpoint = toolkit.load_checkpoint( context, checkpoint, usage_event, ) try: results = checkpoint.run() except Exception as e: toolkit.exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>") if not results["success"]: cli_message("Validation failed!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(1) cli_message("Validation succeeded!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(0)
def validation_operator_run(name, run_name, validation_config_file, suite, directory): # Note though the long lines here aren't pythonic, they look best if Click does the line wraps. """ Run a validation operator against some data. There are two modes to run this command: 1. Interactive (good for development): Specify the name of the validation operator using the --name argument and the name of the expectation suite using the --suite argument. The cli will help you specify the batch of data that you want to validate interactively. 2. Non-interactive (good for production): Use the `--validation_config_file` argument to specify the path of the validation configuration JSON file. This file can be used to instruct a validation operator to validate multiple batches of data and use multiple expectation suites to validate each batch. Learn how to create a validation config file here: https://great-expectations.readthedocs.io/en/latest/command_line.html#great-expectations-validation-operator-run-validation-config-file-validation-config-file-path This command exits with 0 if the validation operator ran and the "success" attribute in its return object is True. Otherwise, the command exits with 1. To learn more about validation operators, go here: https://great-expectations.readthedocs.io/en/latest/features/validation.html#validation-operators """ try: context = DataContext(directory) except ge_exceptions.ConfigNotFoundError as err: cli_message("Failed to process <red>{}</red>".format(err.message)) sys.exit(1) try: if validation_config_file is not None: try: with open(validation_config_file) as f: validation_config = json.load(f) except (OSError, json_parse_exception) as e: cli_message( f"Failed to process the --validation_config_file argument: <red>{e}</red>" ) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) validation_config_error_message = _validate_valdiation_config( validation_config) if validation_config_error_message is not None: cli_message( "<red>The validation config in {:s} is misconfigured: {:s}</red>" .format(validation_config_file, validation_config_error_message)) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) else: if suite is None: cli_message(""" Please use --suite argument to specify the name of the expectation suite. Call `great_expectation suite list` command to list the expectation suites in your project. """) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(0) suite = toolkit.load_expectation_suite( context, suite, "cli.validation_operator.run") if name is None: cli_message(""" Please use --name argument to specify the name of the validation operator. Call `great_expectation validation-operator list` command to list the operators in your project. """) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) else: if name not in context.list_validation_operator_names(): cli_message(f""" Could not find a validation operator {name}. Call `great_expectation validation-operator list` command to list the operators in your project. """) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) batch_kwargs = None cli_message(""" Let us help you specify the batch of data your want the validation operator to validate.""" ) try: data_source = toolkit.select_datasource(context) except ValueError as ve: cli_message("<red>{}</red>".format(ve)) send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) if not data_source: cli_message("<red>No datasources found in the context.</red>") send_usage_message( data_context=context, event="cli.validation_operator.run", success=False, ) sys.exit(1) if batch_kwargs is None: ( datasource_name, batch_kwargs_generator, data_asset, batch_kwargs, ) = get_batch_kwargs( context, datasource_name=data_source.name, batch_kwargs_generator_name=None, data_asset_name=None, additional_batch_kwargs=None, ) validation_config = { "validation_operator_name": name, "batches": [{ "batch_kwargs": batch_kwargs, "expectation_suite_names": [suite.expectation_suite_name], }], } try: validation_operator_name = validation_config[ "validation_operator_name"] batches_to_validate = [] for entry in validation_config["batches"]: for expectation_suite_name in entry["expectation_suite_names"]: batch = context.get_batch(entry["batch_kwargs"], expectation_suite_name) batches_to_validate.append(batch) if run_name is None: run_name = datetime.datetime.now( datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") run_id = RunIdentifier(run_name=run_name) if suite is None: results = context.run_validation_operator( validation_operator_name, assets_to_validate=batches_to_validate, run_id=run_id, ) else: if suite.evaluation_parameters is None: results = context.run_validation_operator( validation_operator_name, assets_to_validate=batches_to_validate, run_id=run_id, ) else: results = context.run_validation_operator( validation_operator_name, assets_to_validate=batches_to_validate, run_id=run_id, evaluation_parameters=suite.evaluation_parameters, ) except (ge_exceptions.DataContextError, OSError, SQLAlchemyError) as e: cli_message("<red>{}</red>".format(e)) send_usage_message(data_context=context, event="cli.validation_operator.run", success=False) sys.exit(1) if not results["success"]: cli_message("Validation failed!") send_usage_message(data_context=context, event="cli.validation_operator.run", success=True) sys.exit(1) else: cli_message("Validation succeeded!") send_usage_message(data_context=context, event="cli.validation_operator.run", success=True) sys.exit(0) except Exception as e: send_usage_message(data_context=context, event="cli.validation_operator.run", success=False) raise e
def init(target_directory, view, usage_stats): """ Initialize a new Great Expectations project. This guided input walks the user through setting up a new project and also onboards a new developer in an existing project. It scaffolds directories, sets up notebooks, creates a project file, and appends to a `.gitignore` file. """ target_directory = os.path.abspath(target_directory) ge_dir = _get_full_path_to_ge_dir(target_directory) cli_message(GREETING) if DataContext.does_config_exist_on_disk(ge_dir): try: if DataContext.is_project_initialized(ge_dir): # Ensure the context can be instantiated cli_message(PROJECT_IS_COMPLETE) except (DataContextError, DatasourceInitializationError) as e: cli_message("<red>{}</red>".format(e.message)) sys.exit(1) try: context = DataContext.create(target_directory, usage_statistics_enabled=usage_stats) cli_message(ONBOARDING_COMPLETE) # TODO if this is correct, ensure this is covered by a test # cli_message(SETUP_SUCCESS) # exit(0) except DataContextError as e: cli_message("<red>{}</red>".format(e.message)) # TODO ensure this is covered by a test exit(5) else: if not click.confirm(LETS_BEGIN_PROMPT, default=True): cli_message(RUN_INIT_AGAIN) # TODO ensure this is covered by a test exit(0) try: context = DataContext.create(target_directory, usage_statistics_enabled=usage_stats) send_usage_message(data_context=context, event="cli.init.create", success=True) except DataContextError as e: # TODO ensure this is covered by a test cli_message("<red>{}</red>".format(e)) try: # if expectations exist, offer to build docs context = DataContext(ge_dir) if context.list_expectation_suites(): if click.confirm(BUILD_DOCS_PROMPT, default=True): build_docs(context, view=view) else: datasources = context.list_datasources() if len(datasources) == 0: datasource_name, data_source_type = add_datasource_impl( context, choose_one_data_asset=True) if not datasource_name: # no datasource was created sys.exit(1) datasources = context.list_datasources() if len(datasources) == 1: datasource_name = datasources[0]["name"] success, suite_name = toolkit.create_expectation_suite( context, datasource_name=datasource_name, additional_batch_kwargs={"limit": 1000}, open_docs=view) if success: cli_message( "A new Expectation suite '{}' was added to your project" .format(suite_name)) cli_message(SETUP_SUCCESS) sys.exit(0) except (DataContextError, ge_exceptions.ProfilerError, IOError, SQLAlchemyError) as e: cli_message("<red>{}</red>".format(e)) sys.exit(1)
def init(target_directory, view, usage_stats): """ Initialize a new Great Expectations project. This guided input walks the user through setting up a new project and also onboards a new developer in an existing project. It scaffolds directories, sets up notebooks, creates a project file, and appends to a `.gitignore` file. """ target_directory = os.path.abspath(target_directory) ge_dir = _get_full_path_to_ge_dir(target_directory) cli_message(GREETING) if DataContext.does_config_exist_on_disk(ge_dir): try: if DataContext.is_project_initialized(ge_dir): # Ensure the context can be instantiated cli_message(PROJECT_IS_COMPLETE) except (DataContextError, DatasourceInitializationError) as e: cli_message("<red>{}</red>".format(e.message)) sys.exit(1) try: context = DataContext.create(target_directory, usage_statistics_enabled=usage_stats) cli_message(ONBOARDING_COMPLETE) # TODO if this is correct, ensure this is covered by a test # cli_message(SETUP_SUCCESS) # exit(0) except DataContextError as e: cli_message("<red>{}</red>".format(e.message)) # TODO ensure this is covered by a test exit(5) else: if not click.confirm(LETS_BEGIN_PROMPT, default=True): cli_message(RUN_INIT_AGAIN) # TODO ensure this is covered by a test exit(0) try: context = DataContext.create(target_directory, usage_statistics_enabled=usage_stats) send_usage_message(data_context=context, event="cli.init.create", success=True) except DataContextError as e: # TODO ensure this is covered by a test cli_message("<red>{}</red>".format(e)) try: # if expectations exist, offer to build docs context = DataContext(ge_dir) if context.list_expectation_suites(): if click.confirm(BUILD_DOCS_PROMPT, default=True): build_docs(context, view=view) else: datasources = context.list_datasources() if len(datasources) == 0: cli_message(SECTION_SEPARATOR) if not click.confirm( "Would you like to configure a Datasource?", default=True): cli_message("Okay, bye!") sys.exit(1) datasource_name, data_source_type = add_datasource_impl( context, choose_one_data_asset=False) if not datasource_name: # no datasource was created sys.exit(1) datasources = context.list_datasources() if len(datasources) == 1: datasource_name = datasources[0]["name"] cli_message(SECTION_SEPARATOR) if not click.confirm( "Would you like to profile new Expectations for a single data asset within your new Datasource?", default=True, ): cli_message( "Okay, exiting now. To learn more about Profilers, run great_expectations profile --help or visit docs.greatexpectations.io!" ) sys.exit(1) ( success, suite_name, profiling_results, ) = toolkit.create_expectation_suite( context, datasource_name=datasource_name, additional_batch_kwargs={"limit": 1000}, flag_build_docs=False, open_docs=False, ) cli_message(SECTION_SEPARATOR) if not click.confirm("Would you like to build Data Docs?", default=True): cli_message( "Okay, exiting now. To learn more about Data Docs, run great_expectations docs --help or visit docs.greatexpectations.io!" ) sys.exit(1) build_docs(context, view=False) if not click.confirm( "\nWould you like to view your new Expectations in Data Docs? This will open a new browser window.", default=True, ): cli_message( "Okay, exiting now. You can view the site that has been created in a browser, or visit docs.greatexpectations.io for more information!" ) sys.exit(1) toolkit.attempt_to_open_validation_results_in_data_docs( context, profiling_results) cli_message(SECTION_SEPARATOR) cli_message(SETUP_SUCCESS) sys.exit(0) except ( DataContextError, ge_exceptions.ProfilerError, OSError, SQLAlchemyError, ) as e: cli_message("<red>{}</red>".format(e)) sys.exit(1)
def _suite_edit(suite, datasource, directory, jupyter, batch_kwargs): batch_kwargs_json = batch_kwargs batch_kwargs = None try: context = DataContext(directory) except ge_exceptions.ConfigNotFoundError as err: cli_message("<red>{}</red>".format(err.message)) return try: suite = load_expectation_suite(context, suite) citations = suite.get_citations(sort=True, require_batch_kwargs=True) if batch_kwargs_json: try: batch_kwargs = json.loads(batch_kwargs_json) if datasource: batch_kwargs["datasource"] = datasource _batch = context.get_batch(batch_kwargs, suite.expectation_suite_name) assert isinstance(_batch, DataAsset) except json_parse_exception as je: cli_message( "<red>Please check that your batch_kwargs are valid JSON.\n{}</red>".format( je ) ) send_usage_message( data_context=context, event="cli.suite.edit", success=False ) sys.exit(1) except ge_exceptions.DataContextError: cli_message( "<red>Please check that your batch_kwargs are able to load a batch.</red>" ) send_usage_message( data_context=context, event="cli.suite.edit", success=False ) sys.exit(1) except ValueError as ve: cli_message( "<red>Please check that your batch_kwargs are able to load a batch.\n{}</red>".format( ve ) ) send_usage_message( data_context=context, event="cli.suite.edit", success=False ) sys.exit(1) elif citations: citation = citations[-1] batch_kwargs = citation.get("batch_kwargs") if not batch_kwargs: cli_message( """ A batch of data is required to edit the suite - let's help you to specify it.""" ) additional_batch_kwargs = None try: data_source = select_datasource(context, datasource_name=datasource) except ValueError as ve: cli_message("<red>{}</red>".format(ve)) send_usage_message( data_context=context, event="cli.suite.edit", success=False ) sys.exit(1) if not data_source: cli_message("<red>No datasources found in the context.</red>") send_usage_message( data_context=context, event="cli.suite.edit", success=False ) sys.exit(1) if batch_kwargs is None: ( datasource_name, batch_kwargs_generator, data_asset, batch_kwargs, ) = get_batch_kwargs( context, datasource_name=data_source.name, batch_kwargs_generator_name=None, generator_asset=None, additional_batch_kwargs=additional_batch_kwargs, ) notebook_name = "{}.ipynb".format(suite.expectation_suite_name) notebook_path = os.path.join( context.root_directory, context.GE_EDIT_NOTEBOOK_DIR, notebook_name ) NotebookRenderer().render_to_disk(suite, notebook_path, batch_kwargs) if not jupyter: cli_message("To continue editing this suite, run <green>jupyter " f"notebook {notebook_path}</green>") payload = edit_expectation_suite_usage_statistics( data_context=context, expectation_suite_name=suite.expectation_suite_name ) send_usage_message( data_context=context, event="cli.suite.edit", event_payload=payload, success=True ) if jupyter: subprocess.call(["jupyter", "notebook", notebook_path]) except Exception as e: send_usage_message( data_context=context, event="cli.suite.edit", success=False ) raise e
def suite_new(suite, directory, empty, jupyter, view, batch_kwargs): """ Create a new Expectation Suite. Great Expectations will choose a couple of columns and generate expectations about them to demonstrate some examples of assertions you can make about your data. If you wish to skip the examples, add the `--empty` flag. """ try: context = DataContext(directory) except ge_exceptions.ConfigNotFoundError as err: cli_message("<red>{}</red>".format(err.message)) return datasource_name = None generator_name = None generator_asset = None try: if batch_kwargs is not None: batch_kwargs = json.loads(batch_kwargs) success, suite_name = create_expectation_suite_impl( context, datasource_name=datasource_name, batch_kwargs_generator_name=generator_name, generator_asset=generator_asset, batch_kwargs=batch_kwargs, expectation_suite_name=suite, additional_batch_kwargs={"limit": 1000}, empty_suite=empty, show_intro_message=False, open_docs=view, ) if success: cli_message( "A new Expectation suite '{}' was added to your project".format( suite_name ) ) if empty: if jupyter: cli_message("""<green>Because you requested an empty suite, we'll open a notebook for you now to edit it! If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n""") _suite_edit(suite_name, datasource_name, directory, jupyter=jupyter, batch_kwargs=batch_kwargs) send_usage_message( data_context=context, event="cli.suite.new", success=True ) else: send_usage_message( data_context=context, event="cli.suite.new", success=False ) except ( ge_exceptions.DataContextError, ge_exceptions.ProfilerError, IOError, SQLAlchemyError, ) as e: cli_message("<red>{}</red>".format(e)) send_usage_message( data_context=context, event="cli.suite.new", success=False ) sys.exit(1) except Exception as e: send_usage_message( data_context=context, event="cli.suite.new", success=False ) raise e
def _suite_edit(suite, datasource, directory, jupyter, batch_kwargs, usage_event): batch_kwargs_json = batch_kwargs batch_kwargs = None context = load_data_context_with_error_handling(directory) try: suite = load_expectation_suite(context, suite) citations = suite.get_citations(require_batch_kwargs=True) if batch_kwargs_json: try: batch_kwargs = json.loads(batch_kwargs_json) if datasource: batch_kwargs["datasource"] = datasource _batch = toolkit.load_batch(context, suite, batch_kwargs) except json_parse_exception as je: cli_message( "<red>Please check that your batch_kwargs are valid JSON.\n{}</red>".format( je ) ) send_usage_message( data_context=context, event=usage_event, success=False ) sys.exit(1) except ge_exceptions.DataContextError: cli_message( "<red>Please check that your batch_kwargs are able to load a batch.</red>" ) send_usage_message( data_context=context, event=usage_event, success=False ) sys.exit(1) except ValueError as ve: cli_message( "<red>Please check that your batch_kwargs are able to load a batch.\n{}</red>".format( ve ) ) send_usage_message( data_context=context, event=usage_event, success=False ) sys.exit(1) elif citations: citation = citations[-1] batch_kwargs = citation.get("batch_kwargs") if not batch_kwargs: cli_message( """ A batch of data is required to edit the suite - let's help you to specify it.""" ) additional_batch_kwargs = None try: data_source = select_datasource(context, datasource_name=datasource) except ValueError as ve: cli_message("<red>{}</red>".format(ve)) send_usage_message( data_context=context, event=usage_event, success=False ) sys.exit(1) if not data_source: cli_message("<red>No datasources found in the context.</red>") send_usage_message( data_context=context, event=usage_event, success=False ) sys.exit(1) if batch_kwargs is None: ( datasource_name, batch_kwargs_generator, data_asset, batch_kwargs, ) = get_batch_kwargs(context, datasource_name=data_source.name, additional_batch_kwargs=additional_batch_kwargs) notebook_name = "edit_{}.ipynb".format(suite.expectation_suite_name) notebook_path = _get_notebook_path(context, notebook_name) SuiteEditNotebookRenderer().render_to_disk(suite, notebook_path, batch_kwargs) if not jupyter: cli_message( f"To continue editing this suite, run <green>jupyter notebook {notebook_path}</green>" ) payload = edit_expectation_suite_usage_statistics( data_context=context, expectation_suite_name=suite.expectation_suite_name ) send_usage_message( data_context=context, event=usage_event, event_payload=payload, success=True ) if jupyter: toolkit.launch_jupyter_notebook(notebook_path) except Exception as e: send_usage_message(data_context=context, event=usage_event, success=False) raise e
def _suite_new( suite: str, directory: str, empty: bool, jupyter: bool, view: bool, batch_kwargs, usage_event: str, ) -> None: # TODO break this up into demo and new context = toolkit.load_data_context_with_error_handling(directory) datasource_name = None generator_name = None generator_asset = None try: if batch_kwargs is not None: batch_kwargs = json.loads(batch_kwargs) success, suite_name = toolkit.create_expectation_suite( context, datasource_name=datasource_name, batch_kwargs_generator_name=generator_name, generator_asset=generator_asset, batch_kwargs=batch_kwargs, expectation_suite_name=suite, additional_batch_kwargs={"limit": 1000}, empty_suite=empty, open_docs=view, ) if success: cli_message( "A new Expectation suite '{}' was added to your project". format(suite_name)) if empty: if jupyter: cli_message( """<green>Because you requested an empty suite, we'll open a notebook for you now to edit it! If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n""") _suite_edit( suite_name, datasource_name, directory, jupyter=jupyter, batch_kwargs=batch_kwargs, usage_event=usage_event, ) send_usage_message(data_context=context, event=usage_event, success=True) else: send_usage_message(data_context=context, event=usage_event, success=False) except ( ge_exceptions.DataContextError, ge_exceptions.ProfilerError, IOError, SQLAlchemyError, ) as e: cli_message("<red>{}</red>".format(e)) send_usage_message(data_context=context, event=usage_event, success=False) sys.exit(1) except Exception as e: send_usage_message(data_context=context, event=usage_event, success=False) raise e
def docs_build(directory, site_name, view=True): """ Build Data Docs for a project.""" context = toolkit.load_data_context_with_error_handling(directory) build_docs(context, site_name=site_name, view=view) send_usage_message(data_context=context, event="cli.docs.build", success=True)
def exit_with_failure_message_and_stats(context: DataContext, usage_event: str, message: str) -> None: cli_message(message) send_usage_message(context, event=usage_event, success=False) sys.exit(1)
def _suite_new( suite: str, directory: str, empty: bool, jupyter: bool, view: bool, batch_kwargs, usage_event: str, ) -> None: # TODO break this up into demo and new context = toolkit.load_data_context_with_error_handling(directory) datasource_name = None generator_name = None data_asset_name = None try: if batch_kwargs is not None: batch_kwargs = json.loads(batch_kwargs) success, suite_name, profiling_results = toolkit.create_expectation_suite( context, datasource_name=datasource_name, batch_kwargs_generator_name=generator_name, data_asset_name=data_asset_name, batch_kwargs=batch_kwargs, expectation_suite_name=suite, additional_batch_kwargs={"limit": 1000}, empty_suite=empty, show_intro_message=False, open_docs=view, ) if success: if empty: if jupyter: cli_message( """<green>Because you requested an empty suite, we'll open a notebook for you now to edit it! If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n""") send_usage_message(data_context=context, event=usage_event, success=True) _suite_edit( suite_name, datasource_name, directory, jupyter=jupyter, batch_kwargs=batch_kwargs, usage_event= "cli.suite.edit", # or else we will be sending `cli.suite.new` which is incorrect suppress_usage_message= True, # dont want actually send usage_message since the function call is not the result of actual usage ) else: send_usage_message(data_context=context, event=usage_event, success=False) except ( ge_exceptions.DataContextError, ge_exceptions.ProfilerError, OSError, SQLAlchemyError, ) as e: cli_message("<red>{}</red>".format(e)) send_usage_message(data_context=context, event=usage_event, success=False) sys.exit(1) except Exception as e: send_usage_message(data_context=context, event=usage_event, success=False) raise e