def docs_list(ctx): """List known Data Docs sites.""" context = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end docs_sites_url_dicts = context.get_docs_sites_urls() try: if len(docs_sites_url_dicts) == 0: cli_message("No Data Docs sites found") else: docs_sites_strings = [ " - <cyan>{}</cyan>: {}".format( docs_site_dict["site_name"], docs_site_dict.get("site_url") or f"site configured but does not exist. Run the following command to build site: great_expectations " f'docs build --site-name {docs_site_dict["site_name"]}', ) for docs_site_dict in docs_sites_url_dicts ] list_intro_string = _build_intro_string(docs_sites_strings) cli_message_list(docs_sites_strings, list_intro_string) toolkit.send_usage_message(data_context=context, event=usage_event_end, success=True) except Exception as e: toolkit.exit_with_failure_message_and_stats( context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return
def clean_data_docs(ctx, site_name=None, all_sites=False): """Delete data docs""" context = ctx.obj.data_context if (site_name is None and all_sites is False) or (site_name and all_sites): toolkit.exit_with_failure_message_and_stats( context, usage_event="cli.docs.clean", message= "<red>Please specify either --all to remove all sites or a specific site using --site_name</red>", ) try: # if site_name is None, context.clean_data_docs(site_name=site_name) # will clean all sites. context.clean_data_docs(site_name=site_name) toolkit.send_usage_message(data_context=context, event="cli.docs.clean", success=True) cli_message("<green>{}</green>".format("Cleaned data docs")) except DataContextError as de: toolkit.exit_with_failure_message_and_stats( context, usage_event="cli.docs.clean", message=f"<red>{de}</red>", )
def checkpoint_script(checkpoint, directory): """ Create a python script to run a checkpoint. (Experimental) Checkpoints can be run directly without this script using the `great_expectations checkpoint run` command. This script is provided for those who wish to run checkpoints via python. """ context = toolkit.load_data_context_with_error_handling(directory) usage_event = "cli.checkpoint.script" # Attempt to load the checkpoint and deal with errors _ = toolkit.load_checkpoint(context, checkpoint, usage_event) script_name = f"run_{checkpoint}.py" script_path = os.path.join(context.root_directory, context.GE_UNCOMMITTED_DIR, script_name) if os.path.isfile(script_path): toolkit.exit_with_failure_message_and_stats( context, usage_event, f"""<red>Warning! A script named {script_name} already exists and this command will not overwrite it.</red> - Existing file path: {script_path}""", ) _write_checkpoint_script_to_disk(context.root_directory, checkpoint, script_path) cli_message( f"""<green>A python script was created that runs the checkpoint named: `{checkpoint}`</green> - The script is located in `great_expectations/uncommitted/run_{checkpoint}.py` - The script can be run with `python great_expectations/uncommitted/run_{checkpoint}.py`""" ) send_usage_message(context, event=usage_event, success=True)
def checkpoint_run(checkpoint, directory): """Run a checkpoint. (Experimental)""" context = toolkit.load_data_context_with_error_handling( directory=directory, from_cli_upgrade_command=False) usage_event = "cli.checkpoint.run" checkpoint: Checkpoint = toolkit.load_checkpoint( context, checkpoint, usage_event, ) try: results = checkpoint.run() except Exception as e: toolkit.exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>") if not results["success"]: cli_message("Validation failed!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(1) cli_message("Validation succeeded!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(0)
def checkpoint_run(ctx, checkpoint): """Run a Checkpoint.""" usage_event: str = "cli.checkpoint.run" context: DataContext = ctx.obj.data_context try: result: CheckpointResult = toolkit.run_checkpoint( context=context, checkpoint_name=checkpoint, usage_event=usage_event, ) except Exception as e: toolkit.exit_with_failure_message_and_stats( context=context, usage_event=usage_event, message=f"<red>{e}</red>", ) return if not result["success"]: cli_message(string="Validation failed!") toolkit.send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(result=result) sys.exit(1) cli_message("Validation succeeded!") toolkit.send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(result=result) sys.exit(0)
def suite_delete(ctx, suite): """ Delete an expectation suite from the expectation store. """ display_not_implemented_message_and_exit() usage_event = "cli.suite.delete" directory = toolkit.parse_cli_config_file_location( config_file_location=ctx.obj.config_file_location).get("directory") context = toolkit.load_data_context_with_error_handling(directory) suite_names = context.list_expectation_suite_names() if not suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, "</red>No expectation suites found in the project.</red>", ) if suite not in suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, f"No expectation suite named {suite} found.") context.delete_expectation_suite(suite) cli_message(f"Deleted the expectation suite named: {suite}") toolkit.send_usage_message(data_context=context, event=usage_event, success=True)
def checkpoint_delete(ctx: click.Context, checkpoint: str) -> None: """Delete a Checkpoint.""" context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: toolkit.delete_checkpoint( context=context, checkpoint_name=checkpoint, usage_event=usage_event_end, assume_yes=ctx.obj.assume_yes, ) send_usage_message( data_context=context, event=usage_event_end, success=True, ) except Exception as e: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return cli_message(f'Checkpoint "{checkpoint}" deleted.') sys.exit(0)
def checkpoint_delete(ctx, checkpoint): """Delete a Checkpoint.""" usage_event: str = "cli.checkpoint.delete" context: DataContext = ctx.obj.data_context try: toolkit.delete_checkpoint( context=context, checkpoint_name=checkpoint, usage_event=usage_event, assume_yes=ctx.obj.assume_yes, ) toolkit.send_usage_message(context, event="cli.checkpoint.delete", success=True) except Exception as e: toolkit.exit_with_failure_message_and_stats( context=context, usage_event=usage_event, message=f"<red>{e}</red>", ) return cli_message(f'Checkpoint "{checkpoint}" deleted.') sys.exit(0)
def docs_clean(ctx, site_name=None, all_sites=False): """ Remove all files from a Data Docs site. This is a useful first step if you wish to completely re-build a site from scratch. """ context = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end if (site_name is None and all_sites is False) or (site_name and all_sites): toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message= "<red>Please specify either --all to clean all sites or a specific site using --site-name</red>", ) try: # if site_name is None, context.clean_data_docs(site_name=site_name) # will clean all sites. context.clean_data_docs(site_name=site_name) toolkit.send_usage_message(data_context=context, event=usage_event_end, success=True) cli_message("<green>{}</green>".format("Cleaned data docs")) except DataContextError as de: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message=f"<red>{de}</red>", )
def docs_build(ctx, site_name=None, no_view=False): """Build Data Docs for a project.""" context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end if site_name is not None and site_name not in context.get_site_names(): toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message= f"<red>The specified site name `{site_name}` does not exist in this project.</red>", ) if site_name is None: sites_to_build = context.get_site_names() else: sites_to_build = [site_name] build_docs( context, usage_stats_event=usage_event_end, site_names=sites_to_build, view=not no_view, assume_yes=ctx.obj.assume_yes, ) toolkit.send_usage_message(data_context=context, event=usage_event_end, success=True)
def datasource_list(ctx: click.Context) -> None: """List known Datasources.""" context = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: datasources = context.list_datasources() cli_message(_build_datasource_intro_string(datasources)) for datasource in datasources: cli_message("") cli_message_dict({ "name": datasource["name"], "class_name": datasource["class_name"], }) send_usage_message( data_context=context, event=usage_event_end, success=True, ) except Exception as e: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return
def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str, usage_event: str) -> None: if checkpoint in context.list_checkpoints(): toolkit.exit_with_failure_message_and_stats( context, usage_event, f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.", )
def _validate_at_least_one_suite_is_listed(context: DataContext, batch: dict, checkpoint_file: str) -> None: batch_kwargs = batch["batch_kwargs"] suites = batch["expectation_suite_names"] if not suites: toolkit.exit_with_failure_message_and_stats( context, "cli.checkpoint.run", f"""<red>A batch has no suites associated with it. At least one suite is required. - Batch: {batch_kwargs} - Please add at least one suite to your checkpoint file: {checkpoint_file}</red>""", )
def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str, usage_event: str) -> None: try: if checkpoint in context.list_checkpoints(): toolkit.exit_with_failure_message_and_stats( context, usage_event, f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.", ) except InvalidTopLevelConfigKeyError as e: toolkit.exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>")
def checkpoint_run(checkpoint, directory): """Run a checkpoint. (Experimental)""" context = toolkit.load_data_context_with_error_handling(directory) usage_event = "cli.checkpoint.run" checkpoint_config = toolkit.load_checkpoint(context, checkpoint, usage_event) checkpoint_file = f"great_expectations/checkpoints/{checkpoint}.yml" # TODO loading batches will move into DataContext eventually batches_to_validate = [] for batch in checkpoint_config["batches"]: _validate_at_least_one_suite_is_listed(context, batch, checkpoint_file) batch_kwargs = batch["batch_kwargs"] for suite_name in batch["expectation_suite_names"]: suite = toolkit.load_expectation_suite(context, suite_name, usage_event) try: batch = toolkit.load_batch(context, suite, batch_kwargs) except (FileNotFoundError, SQLAlchemyError, OSError, DataContextError) as e: toolkit.exit_with_failure_message_and_stats( context, usage_event, f"""<red>There was a problem loading a batch: - Batch: {batch_kwargs} - {e} - Please verify these batch kwargs in the checkpoint file: `{checkpoint_file}`</red>""", ) batches_to_validate.append(batch) try: results = context.run_validation_operator( checkpoint_config["validation_operator_name"], assets_to_validate=batches_to_validate, # TODO prepare for new RunID - checkpoint name and timestamp # run_id=RunID(checkpoint) ) except DataContextError as e: toolkit.exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>") if not results["success"]: cli_message("Validation failed!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(1) cli_message("Validation succeeded!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(0)
def suite_delete(ctx: click.Context, suite: str) -> None: """ Delete an Expectation Suite from the Expectation Store. """ context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: suite_names: List[str] = context.list_expectation_suite_names() except Exception as e: send_usage_message( data_context=context, event=usage_event_end, success=False, ) raise e if not suite_names: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, suppress_usage_message=False, message="<red>No expectation suites found in the project.</red>", ) if suite not in suite_names: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, suppress_usage_message=False, message=f"<red>No expectation suite named {suite} found.</red>", ) if not ( ctx.obj.assume_yes or toolkit.confirm_proceed_or_exit( exit_on_no=False, data_context=context, usage_stats_event=usage_event_end ) ): cli_message(string=f"Suite `{suite}` was not deleted.") sys.exit(0) context.delete_expectation_suite(suite) cli_message(string=f"Deleted the expectation suite named: {suite}") send_usage_message( data_context=context, event=usage_event_end, success=True, )
def checkpoint_script(ctx: click.Context, checkpoint: str) -> None: """ Create a python script to run a Checkpoint. Checkpoints can be run directly without this script using the `great_expectations Checkpoint run` command. This script is provided for those who wish to run Checkpoints via python. """ context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end toolkit.validate_checkpoint(context=context, checkpoint_name=checkpoint, usage_event=usage_event_end) script_name: str = f"run_{checkpoint}.py" script_path: str = os.path.join(context.root_directory, context.GE_UNCOMMITTED_DIR, script_name) if os.path.isfile(script_path): toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message= f"""<red>Warning! A script named {script_name} already exists and this command will not overwrite it.</red> - Existing file path: {script_path}""", ) _write_checkpoint_script_to_disk( context_directory=context.root_directory, checkpoint_name=checkpoint, script_path=script_path, ) cli_message( f"""<green>A python script was created that runs the Checkpoint named: `{checkpoint}`</green> - The script is located in `great_expectations/uncommitted/run_{checkpoint}.py` - The script can be run with `python great_expectations/uncommitted/run_{checkpoint}.py`""" ) send_usage_message( data_context=context, event=usage_event_end, success=True, )
def suite_delete(suite, directory): """ Delete an expectation suite from the expectation store. """ usage_event = "cli.suite.delete" context = toolkit.load_data_context_with_error_handling(directory) suite_names = context.list_expectation_suite_names() if not suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, "</red>No expectation suites found in the project.</red>") if suite not in suite_names: toolkit.exit_with_failure_message_and_stats( context, usage_event, f"No expectation suite named {suite} found.") context.delete_expectation_suite(suite) cli_message(f"Deleted the expectation suite named: {suite}") send_usage_message(data_context=context, event=usage_event, success=True)
def _checkpoint_new(ctx, checkpoint_name, jupyter): context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: _verify_checkpoint_does_not_exist(context, checkpoint_name, usage_event_end) # Create notebook on disk notebook_name = f"edit_checkpoint_{checkpoint_name}.ipynb" notebook_file_path = _get_notebook_path(context, notebook_name) checkpoint_new_notebook_renderer = CheckpointNewNotebookRenderer( context=context, checkpoint_name=checkpoint_name ) checkpoint_new_notebook_renderer.render_to_disk( notebook_file_path=notebook_file_path ) if not jupyter: cli_message( f"To continue editing this Checkpoint, run <green>jupyter notebook {notebook_file_path}</green>" ) send_usage_message( data_context=context, event=usage_event_end, success=True, ) if jupyter: cli_message( """<green>Because you requested to create a new Checkpoint, we'll open a notebook for you now to edit it! If you wish to avoid this you can add the `--no-jupyter` flag.</green>\n\n""" ) toolkit.launch_jupyter_notebook(notebook_file_path) except Exception as e: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return
def datasource_new(ctx: click.Context, name: str, jupyter: bool) -> None: """Add a new Datasource to the data context.""" context: DataContext = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: _datasource_new_flow( context, usage_event_end=usage_event_end, datasource_name=name, jupyter=jupyter, ) except Exception as e: toolkit.exit_with_failure_message_and_stats( data_context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return
def store_list(ctx): """List active Stores.""" context = ctx.obj.data_context usage_event_end: str = ctx.obj.usage_event_end try: stores = context.list_active_stores() cli_message(f"{len(stores)} active Stores found:") for store in stores: cli_message("") cli_message_dict(store) toolkit.send_usage_message(data_context=context, event=usage_event_end, success=True) except Exception as e: toolkit.exit_with_failure_message_and_stats( context=context, usage_event=usage_event_end, message=f"<red>{e}</red>", ) return
def checkpoint_run(checkpoint, directory): """Run a checkpoint. (Experimental)""" usage_event = "cli.checkpoint.run" context = toolkit.load_data_context_with_error_handling( directory=directory, from_cli_upgrade_command=False) ge_config_version = context.get_config().config_version if ge_config_version >= 3: cli_message( f"""<red>The `checkpoint run` CLI command is not yet implemented for GE config versions >= 3.</red>""" ) send_usage_message(context, usage_event, success=False) sys.exit(1) checkpoint: Checkpoint = toolkit.load_checkpoint( context, checkpoint, usage_event, ) try: results = checkpoint.run() except Exception as e: toolkit.exit_with_failure_message_and_stats(context, usage_event, f"<red>{e}</red>") if not results["success"]: cli_message("Validation failed!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(1) cli_message("Validation succeeded!") send_usage_message(context, event=usage_event, success=True) print_validation_operator_results_details(results) sys.exit(0)