def node( title: str = Option("", "--title", "-n", help=_name_help), component: str = Option("", "-c", "--component", help=_component_help), location: Path = Argument(None), ): """Add a new node to a graph patterns create node --name='My Node' mynode.py """ if component and location: abort("Specify either a component or a node location, not both") if component: ids = IdLookup(find_nearest_graph=True) GraphConfigEditor(ids.graph_file_path).add_component_uses( component_key=component).write() sprint(f"[success]Added component {component} to graph") return if not location: sprint("[info]Nodes can be python files like [code]ingest.py") sprint("[info]Nodes can be sql files like [code]aggregate.sql") sprint( "[info]You also can add a subgraph like [code]processor/graph.yml") message = "Enter a name for the new node file" location = prompt_path(message, exists=False) if location.exists(): abort(f"Cannot create node: {location} already exists") ids = IdLookup(node_file_path=location, find_nearest_graph=True) # Update the graph yaml node_file = "/".join(location.absolute().relative_to( ids.graph_directory).parts) node_title = title or (location.parent.name if location.name == "graph.yml" else location.stem) with abort_on_error("Adding node failed"): editor = GraphConfigEditor(ids.graph_file_path) editor.add_node( title=node_title, node_file=node_file, id=str(random_node_id()), ) # Write to disk last to avoid partial updates if location.suffix == ".py": location.write_text(_PY_FILE_TEMPLATE) elif location.suffix == ".sql": location.write_text(_SQL_FILE_TEMPLATE) elif location.name == "graph.yml": location.parent.mkdir(exist_ok=True, parents=True) GraphConfigEditor(location, read=False).set_name(node_title).write() else: abort("Node file must be graph.yml or end in .py or .sql") editor.write() sprint(f"\n[success]Created node [b]{location}") sprint( f"\n[info]Once you've edited the node and are ready to run the graph, " f"use [code]patterns upload")
def trigger( organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), graph: Path = Option(None, exists=True, help=_graph_help), graph_version_id: str = Option("", help=_graph_version_id_help), type: TypeChoices = Option(TypeChoices.pubsub, hidden=True), node: Path = Argument(..., exists=True, help=_node_help), ): """Trigger a node on a deployed graph to run immediately""" ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, node_file_path=node, explicit_graph_version_id=graph_version_id, find_nearest_graph=True, ) with abort_on_error("Error triggering node"): trigger_node( ids.node_id, ids.graph_version_id, ids.environment_id, execution_type=type, ) sprint(f"[success]Triggered node {node}")
def pull( organization: str = Option("", "-o", "--organization", help=_organization_help), graph_version_id: str = Option("", help=_graph_version_id_help), force: bool = Option(False, "-f", "--force", help=_force_help), graph: Path = Argument(None, exists=True, help=_pull_graph_help), ): """Update the code for the current graph""" ids = IdLookup( organization_name=organization, explicit_graph_version_id=graph_version_id, explicit_graph_path=graph, ) with abort_on_error("Error downloading graph"): b = io.BytesIO(download_graph_zip(ids.graph_version_id)) editor = GraphDirectoryEditor(ids.graph_file_path, overwrite=force) with abort_on_error("Error downloading graph"): try: with ZipFile(b, "r") as zf: editor.add_node_from_zip("graph.yml", "graph.yml", zf) except FileOverwriteError as e: sprint(f"[error]{e}") sprint( "[info]Run this command with --force to overwrite local files") raise typer.Exit(1) sprint(f"[success]Pulled graph content")
def deploy( environment: str = Option("", "-e", "--environment", help=_environment_help), organization: str = Option("", "-o", "--organization", help=_organization_help), graph: Path = Option(None, help=_graph_help), graph_version_id: str = Option("", help=_graph_version_id_help), ): """Deploy a previously uploaded graph version You can specify either '--graph-version-id' to deploy a specific version, or '--graph' to deploy the latest uploaded version of a graph. """ ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, explicit_graph_version_id=graph_version_id, ) with abort_on_error("Deploy failed"): deploy_graph_version(ids.graph_version_id, ids.environment_id) sprint(f"[success]Graph deployed.")
def test_find_graph_from_node(tmp_path: Path): setup_graph_files( tmp_path, { "graph.yml": """ nodes: - node_file: node1.py - node_file: dir/node2.py - node_file: sub/graph.yml """, "node1.py": "t3=OutputTable", "dir/node2.py": "t1=OutputTable", "sub/graph.yml": """ nodes: - node_file: node3.py """, "sub/node3.py": "t2=OutputTable", }, ) for p in [ tmp_path / "dir" / "node1.py", tmp_path / "sub" / "node2.py", tmp_path / "node3.py", ]: actual = IdLookup(node_file_path=p).graph_file_path assert actual == tmp_path / "graph.yml"
def upload( deploy: bool = Option(True, "--deploy/--no-deploy", help=_deploy_help), organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), graph: Path = Argument(None, exists=True, help=_graph_help), publish_component: bool = Option(False, help=_component_help), ): """Upload a new version of a graph to Patterns""" ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, ) with abort_on_error("Upload failed"): resp = upload_graph_version( ids.graph_file_path, ids.organization_id, add_missing_node_ids=not publish_component, ) graph_version_id = resp["uid"] ui_url = resp["ui_url"] sprint(f"\n[success]Uploaded new graph version with id [b]{graph_version_id}") errors = resp.get("errors", []) if publish_component: errors = [ e for e in errors if not e["message"].startswith("Top level input is not connected") and not ( e["message"].startswith("Parameter") and e["message"].endswith("has no default or value") ) ] if errors: sprint(f"[error]Graph contains the following errors:") for error in errors: sprint(f"\t[error]{error}") if publish_component: with abort_on_error("Error creating component"): resp = create_graph_component(graph_version_id) resp_org = resp["organization"]["slug"] resp_version = resp["version_name"] resp_component = resp["component"]["slug"] resp_id = resp["uid"] sprint( f"[success]Published graph component " f"[b]{resp_org}/{resp_component}[/b] " f"with version [b]{resp_version}[/b] " f"at id [b]{resp_id}" ) elif deploy: with abort_on_error("Deploy failed"): deploy_graph_version(graph_version_id, ids.environment_id) sprint(f"[success]Graph deployed") sprint(f"\n[info]Visit [code]{ui_url}[/code] to view your graph")
def environments( organization: str = Option("", "--organization", "-o", help=_organization_help), print_json: bool = Option(False, "--json", help=_json_help), ): """List environments""" ids = IdLookup(organization_name=organization) with abort_on_error("Error listing environments"): es = list(paginated_environments(ids.organization_id)) _print_objects(es, print_json)
def graphs( organization: str = Option("", help=_organization_help), print_json: bool = Option(False, "--json", help=_json_help), ): """List graphs""" ids = IdLookup(organization_name=organization) with abort_on_error("Error listing graphs"): gs = list(paginated_graphs(ids.organization_id)) _print_objects(gs, print_json)
def clone( organization: str = Option("", "-o", "--organization", help=_organization_help), graph: str = Option("", help=_graph_help), graph_version_id: str = Option("", "-v", "--version", help=_graph_version_id_help), component: str = Option("", "--component", help=_component_help), directory: Path = Argument(None, exists=False, help=_graph_help), ): """Download the code for a graph""" if not graph and not directory and not component: if graph_version_id: abort( f"Missing graph directory argument." f"\ntry [code]patterns clone -v {graph_version_id} new_graph") else: abort(f"Missing graph argument." f"\ntry [code]patterns clone graph-to-clone") component_match = COMPONENT_RE.fullmatch(component) if component and not component_match: abort( "Invalid component version. Must be in the form organization/component@v1" ) component_name = component_match.group(2) if component_match else None ids = IdLookup( organization_name=organization, explicit_graph_name=graph or component_name or directory.name, explicit_graph_version_id=graph_version_id, ) if not directory: if component: directory = Path(component_name) elif graph: directory = Path(graph) elif graph_version_id: with abort_on_error("Error"): directory = Path(ids.graph_name) else: abort("Specify --graph, --graph-version-id, or a directory") with abort_on_error("Error cloning graph"): if component: content = download_component_zip(component) else: content = download_graph_zip(ids.graph_version_id) editor = GraphDirectoryEditor(directory, overwrite=False) with ZipFile(io.BytesIO(content), "r") as zf: editor.add_node_from_zip("graph.yml", "graph.yml", zf) sprint(f"[success]Cloned graph into {directory}")
def webhooks( print_json: bool = Option(False, "--json", help=_json_help), organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), graph: Path = Argument(None, exists=True, help=_graph_help), ): """List webhook urls for a graph""" ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, ) with abort_on_error("Could not get webhook data"): data = list(paginated_webhook_urls(ids.environment_id, ids.graph_id)) _print_objects(data, print_json, headers=["name", "node_id", "webhook_url"])
def logs( print_json: bool = Option(False, "--json", help=_json_help), organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), node: Path = Argument(..., exists=True, help=_node_help), ): """List execution logs for a node""" ids = IdLookup( environment_name=environment, organization_name=organization, node_file_path=node, ) with abort_on_error("Could not list logs"): events = list( paginated_execution_events(ids.environment_id, ids.graph_id, ids.node_id) ) _print_objects(events, print_json)
def output( print_json: bool = Option(False, "--json", help=_json_help), organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), graph: Path = Option("", "-g", "--graph-path", help=_graph_help), store_name: str = Argument(..., exists=True, help=_store_name_help), ): """List data sent to an output port of a from the most recent run of a node""" ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, ) with abort_on_error("Could not get node data"): data = list( paginated_output_data(ids.environment_id, ids.graph_id, store_name) ) _print_objects(data, print_json)
def delete( graph_id: str = Option(""), force: bool = Option(False, "-f", "--force", help=_force_help), graph: Path = Argument(None, exists=True, help=_graph_help), ): """Delete a graph from the Patterns studio. This will not delete any files locally. """ ids = IdLookup( explicit_graph_path=graph, explicit_graph_id=graph_id, ) with abort_on_error("Deleting graph failed"): if not force: Confirm.ask(f"Delete graph {ids.graph_name}?") delete_graph(ids.graph_id) sprint(f"[success]Graph deleted from Patterns studio.")
def webhook( explicit_graph: Path = Option(None, "--graph", "-g", exists=True, help=_graph_help), name: str = Argument(..., help=_webhook_name_help), ): """Add a new webhook node to a graph""" ids = IdLookup(explicit_graph_path=explicit_graph) with abort_on_error("Adding webhook failed"): editor = GraphConfigEditor(ids.graph_file_path) editor.add_webhook(name, id=random_node_id()) editor.write() sprint(f"\n[success]Created webhook [b]{name}") sprint( f"\n[info]Once you've deployed the graph, use " f"[code]patterns list webhooks[/code] to get the url of the webhook")
def login(): """Log in to your Patterns account""" with abort_on_error("Login failed"): login_service.login() ids = IdLookup(ignore_local_cfg=True) with abort_on_error("Fetching account failed"): update_devkit_config(organization_id=ids.organization_id, environment_id=ids.environment_id) with abort_on_error("Fetching user profile failed"): profile = me() sprint( f"\n[success]Logged in to Patterns organization [b]{ids.organization_name}[/b] " f"as [b]{profile['username']}[/b] ([b]{profile['email']}[/b])") sprint(f"\n[info]Your login information is stored at " f"{get_devkit_config_path().as_posix()}") sprint(f"\n[info]If you want to create a new graph, run " f"[code]patterns create graph[/code] get started")
def config( organization: str = Option("", "-o", "--organization", help=_config_help), environment: str = Option("", "-e", "--environment", help=_environment_help), ): """Change the default values used by other commands""" ids = IdLookup( organization_name=organization, environment_name=environment, ignore_cfg_environment=True, ) if organization: ids.cfg.organization_id = ids.organization_id if environment: ids.cfg.environment_id = ids.environment_id write_devkit_config(ids.cfg) sprint(f"[info]Your patterns config is located at " f"[code]{get_devkit_config_path().as_posix()}") t = Table(show_header=False) try: name = get_organization_by_id(ids.organization_id)["name"] t.add_row("organization", name) except Exception: t.add_row("organization_id", ids.organization_id) try: name = get_environment_by_id(ids.environment_id)["name"] t.add_row("environment", name) except Exception: t.add_row("environment_id", ids.environment_id) if ids.cfg.auth_server: t.add_row("auth_server.domain", ids.cfg.auth_server.domain) t.add_row("auth_server.audience", ids.cfg.auth_server.audience) t.add_row("auth_server.devkit_client_id", ids.cfg.auth_server.devkit_client_id) sprint(t)