def make(cli: click.Group):
@cli.command(name="search")
@click.argument("query", type=click.STRING)
@click.pass_obj
def search(ctx, query):
ctx = config.getctx(ctx)
r = ctx.session.get(
"search",
params={"q": query},
)
if not r.ok:
exit_with(handle_request_error(r))
for result in r.json()["hits"]["hits"]:
click.secho(
f'{result["_source"]["owner_name"]}/', fg="cyan", bold=True, nl=False
)
click.secho(
f'{result["_source"]["workspace_name"]}',
fg="cyan",
bold=True,
nl=False,
)
click.secho(result["_source"]["path"])
cli.add_command(search)
def main():
_add_project_modules_to_sys_path()
commands = Group(no_args_is_help=True)
for command in SUPPORTED_COMMANDS:
commands.add_command(command, command.name)
click.CommandCollection(sources=[commands])()
def make(cli: click.Group):
@click.command(name="login")
@click.option("--access-key", prompt=True)
@click.option("--secret-key", prompt=True)
@click.pass_obj
def login(ctx, access_key, secret_key):
ctx = config.getctx(ctx)
r = ctx.session.get("user/me", auth=(access_key, secret_key))
if r.ok:
click.echo(click.style("Login success", fg="green", bold=True))
ctx.config.access_key = access_key
ctx.config.secret_key = secret_key
config.save(ctx)
else:
exit_with(handle_request_error(r))
@click.command(name="info")
@click.pass_obj
def me(ctx):
ctx = config.getctx(ctx)
r1 = ctx.session.get("user/me")
if not r1.ok:
exit_with(handle_request_error(r1))
out = r1.json()
out.update(ctx.config.dict())
out.update({"configPath": ctx.configPath})
exit_with(out)
cli.add_command(login)
cli.add_command(me)
def register_to(self, group: click.Group) -> None:
"""
Registers all commands to the given group.
:param group: Click group to register the commands to
"""
for command in self.commands:
group.add_command(command)
def add_plugin_commands(command_group: click.Group) -> None:
"""
Add commands provided by all plugins to the given command group. Each command is
added with a name that is equal to the plugin name.
"""
for plugin in plugins.iter_installed():
if isinstance(plugin.command, click.Command):
plugin.command.name = plugin.name
command_group.add_command(plugin.command)
def maybe_add_run_to_cli(cli: click.Group) -> None:
if "run" not in cli.commands:
if state.file or state.module:
obj = get_typer_from_state()
if obj:
obj._add_completion = False
click_obj = typer.main.get_command(obj)
click_obj.name = "run"
if not click_obj.help:
click_obj.help = "Run the provided Typer app."
cli.add_command(click_obj)
def make(cli: click.Group):
@cli.group(name="token", cls=ClickAliasedGroup, aliases=["t"])
def token():
pass
@token.command(name="fetch", aliases=["f"])
@click.argument("workspaces", type=click.STRING, nargs=-1)
@click.pass_obj
def create_token(ctx, workspaces):
if len(workspaces) == 0:
return
r = ctx["session"].post("token/search",
json={"search_terms": workspaces})
if r.ok:
response = schemas.S3TokenSearchResponse(**r.json())
for wrapper in response.tokens:
node = wrapper.node
token = wrapper.token
click.secho(
f"Credentials for {[w.name for w in token.workspaces]} @ {node.api_url}",
fg="green",
)
click.secho(
f"Expires in {token.expiration - datetime.utcnow()}\n",
fg="yellow")
click.secho(f"export AWS_ACCESS_KEY_ID={token.access_key_id}")
click.secho(
f"export AWS_SECRET_ACCESS_KEY={token.secret_access_key}")
click.secho(
f"export AWS_SESSION_TOKEN={token.session_token}\n")
else:
exit_with(handle_request_error(r))
@token.command(name="list", aliases=["l", "ls"])
@click.pass_obj
def list_token(ctx):
r = ctx["session"].get("token")
exit_with(handle_request_error(r))
@token.command(name="delete", aliases=["d"])
@click.option("--all", is_flag=True)
@click.argument("token_id", required=False)
@click.pass_obj
def delete_token(ctx, all, token_id):
if all:
r = ctx["session"].delete("token")
else:
r = ctx["session"].delete(f"token/{token_id}")
exit_with(handle_request_error(r))
cli.add_command(token)
def add_command(self, cmd, name=None):
""" Hook the added command and put the group options on the command """
Group.add_command(self, cmd, name=name)
cmd.require_project = '@@ignore_check@@' not in (cmd.callback.__doc__
or "")
# add the group parameters to the command
for param in self.params:
cmd.params.append(param)
# hook the command's invoke with our own
cmd.invoke = self.build_command_invoke(cmd.invoke)
self.invoke_without_command = True
def make(cli: click.Group):
@cli.group(name="index")
def index():
pass
@index.command(name="delete")
@click.argument("root_id", type=click.STRING)
@click.pass_obj
def delete(ctx, root_id):
r = config.getctx(ctx).session.delete(f"root/{root_id}/index")
exit_with(handle_request_error(r))
@index.command(name="create")
@click.argument("root_id", type=click.STRING)
@click.pass_obj
def create(ctx, root_id):
ctx = config.getctx(ctx)
r = ctx.session.post(f"root/{root_id}/index")
r2 = ctx.session.get("info")
if r.ok and r2.ok:
data = r.json()
infodata = r2.json()
root_id = data["root_id"]
bucket = data["root"]["bucket"]
prefix = posixpath.join(data["root"]["base_path"], "/").lstrip("/")
endpoint_base = infodata["public_address"]
click.secho(
"To notify Workspaces of updates, configure your MinIO instance using these commands.\n",
fg="yellow",
)
for c in [
# wehook ID should come from ROOT, not index. You only want to subscribe to events once.
f"export ALIAS=local",
f"mc admin config set $ALIAS notify_webhook:{root_id} endpoint={endpoint_base}/api/minio/events enable=on",
f"mc event add $ALIAS/{bucket} arn:minio:sqs::{root_id}:webhook {f'--prefix {prefix}' if prefix else ''} --event delete,put",
]:
click.secho("\t" + c, fg="blue", bold=True)
click.echo("")
else:
exit_with(handle_request_error(r))
cli.add_command(index)
def make(cli: click.Group):
@click.command(name="login")
@click.argument("email")
@click.option("--password", prompt=True, hide_input=True)
@click.pass_obj
def login(ctx, email, password):
conf = ctx["config"]
r = ctx["session"].post(
"auth/jwt/login",
{
"username": email,
"password": password,
},
)
if r.ok:
token = r.json()["access_token"]
click.echo(click.style("Login success", fg="green", bold=True))
conf.token = token
save_config(conf, ctx["configPath"])
else:
exit_with(handle_request_error(r))
@click.command(name="register")
@click.argument("email")
@click.argument("username")
@click.option("--password", prompt=True, hide_input=True)
@click.pass_obj
def register(ctx, email, password, username):
r = ctx["session"].post(
"auth/register",
json={
"email": email,
"username": username,
"password": password,
},
)
exit_with(handle_request_error(r))
@click.command(name="info")
@click.pass_obj
def me(ctx):
r = ctx["session"].get("me")
exit_with(handle_request_error(r))
cli.add_command(login)
cli.add_command(register)
cli.add_command(me)
def add_commands(cli: click.Group) -> None:
cli.add_command(print_parser)
def add_commands(cli: click.Group) -> None:
cli.add_command(print_model)
def make(cli: click.Group):
@cli.group(name="workspace", cls=ClickAliasedGroup, aliases=["w"])
def workspace():
pass
@workspace.command(name="list", aliases=["ls", "l"])
@click.option("--name", type=click.STRING, required=False)
@click.option("--like", type=click.STRING, required=False)
@click.option("--public", is_flag=True)
@click.pass_obj
def list_workspaces(ctx, name, like, public):
params = {"public": public}
if name:
params["name"] = name
if like:
params["like"] = like
r = ctx["session"].get("workspace", params=params)
if r.ok:
for ws in r.json():
scope = ws["root"]["root_type"]
click.secho(f"[{ws['created']}] ", fg="green", nl=False)
click.secho(f"{ws['id']} ", fg="yellow", nl=False)
click.secho(
f"{ws['owner']['username']}/{ws['name']}/ ",
fg="cyan",
bold=True,
nl=False,
)
click.secho(f"({scope})", fg="bright_black")
else:
exit_with(handle_request_error(r))
@workspace.command(name="create", aliases=["c"])
@click.argument("name")
@click.option("--public/--private", default=False, is_flag=True)
@click.option("--unmanaged", default=False, is_flag=True)
@click.option("--node-name", type=click.STRING, default=None)
@click.pass_obj
def create_workspace(ctx, name, public, unmanaged, node_name):
r = ctx["session"].post(
"workspace",
json={
"name": name,
"public": public,
"unmanaged": unmanaged,
"node_name": node_name,
},
)
exit_with(handle_request_error(r))
@workspace.command(name="delete")
@click.argument("workspace_id", type=click.STRING)
@click.pass_obj
def delete_workspace(ctx, workspace_id):
r = ctx["session"].delete(f"workspace/{workspace_id}")
exit_with(handle_request_error(r))
@workspace.command(name="share", aliases=["s"])
@click.argument("workspace", type=click.STRING)
@click.argument("sharee", type=click.STRING)
@click.option(
"--permission",
type=click.Choice(schemas.ShareType),
default=schemas.ShareType.READ.value,
)
@click.option("--expire", type=click.DateTime())
@click.pass_obj
def create_workspace_share(ctx, workspace, sharee, permission, expire):
body = {
"workspace": workspace,
"sharee": sharee,
"permission": permission,
}
if expire:
body["expiration"] = expire
r = ctx["session"].post(
"workspace/share",
json=body,
)
exit_with(handle_request_error(r))
cli.add_command(workspace)
@workspace.command(name="index")
@click.argument("workspace_id", type=click.STRING)
@click.option(
"--minio-mount",
type=click.Path(dir_okay=True, exists=True),
help="Path to minio mount on local disk",
)
@click.pass_obj
def index_workspace(ctx, workspace_id, minio_mount):
# Dynamic, expensive imports
from workspacesio.common import producers
ctx = config.getctx(ctx)
r = ctx.session.get(f"workspace/{workspace_id}")
if not r.ok:
exit_with(handle_request_error(r))
w = schemas.WorkspaceDB(**r.json())
r = ctx.session.post(f"workspace/{w.id}/crawl")
if not r.ok:
exit_with(handle_request_error(r))
data = indexing_schemas.WorkspaceCrawlRoundResponse(**r.json())
startfrom = data.crawl_round.last_indexed_key or ""
root = data.root_credentials.root
node = data.root_credentials.node
for batch in producers.minio_buffer_objects(
producers.minio_recursive_generate_objects(
node=node,
root=root,
workspace=w,
after=startfrom,
),
buffer_size=100,
):
documents: List[indexing_schemas.IndexDocumentBase] = []
for obj in batch:
before = datetime.datetime.utcnow()
obj.time = "ar"
doc = producers.minio_transform_object(workspace=w,
root=root,
obj=obj)
success, failed = producers.additional_indexes(root=root,
workspace=w,
doc=doc,
node=node)
delta = str(
int((datetime.datetime.utcnow() - before).total_seconds() *
1000)).ljust(4)
click.secho(
f"ms={delta} workspace={w.name} analysis={','.join(success)} path={doc.path}",
fg="red" if len(failed) else "green",
)
documents.append(doc)
payload = indexing_schemas.IndexBulkAdd(
documents=documents,
workspace_id=w.id,
last_indexed_key=documents[-1].path,
succeeded=False,
)
r = ctx.session.post(
f"workspace/{w.id}/bulk_index",
data=payload.json(),
)
r.raise_for_status()
exit_with(
handle_request_error(
ctx.session.post(
f"workspace/{w.id}/bulk_index",
data=indexing_schemas.IndexBulkAdd(
documents=[],
workspace_id=w.id,
succeeded=True,
).json(),
)))
def add_subcommands(
_cli: click.Group,
):
_cli.add_command(translate)
_cli.add_command(add_price_column)
def add_commands(command_group: click.Group) -> None:
command_group.add_command(start)
command_group.add_command(stop)
command_group.add_command(restart)
command_group.add_command(reboot)
command_group.add_command(init)
command_group.add_command(createuser)
command_group.add_command(importdemocourse)
command_group.add_command(settheme)
command_group.add_command(dc_command)
command_group.add_command(run)
command_group.add_command(copyfrom)
command_group.add_command(bindmount_command)
command_group.add_command(execute)
command_group.add_command(logs)
command_group.add_command(status)
def make(cli: click.Group):
@cli.group(name="node", cls=ClickAliasedGroup, aliases=["n"])
def node():
pass
@node.command(name="list", aliases=["l", "ls"])
@click.pass_obj
def ls(ctx):
r = ctx["session"].get("node")
if r.ok:
for node in r.json():
api_url = node["api_url"]
click.secho(f"[{node['created']}] ", fg="green", nl=False)
click.secho(f"{node['id']} ", fg="yellow", nl=False)
click.secho(f"{api_url} ", fg="bright_black", nl=False)
click.secho(f"{node['name']}", fg="cyan", bold=True)
else:
exit_with(handle_request_error(r))
@node.command(name="delete", aliases=["d"])
@click.argument("node_id", type=click.STRING, required=True)
@click.pass_obj
def rm(ctx, node_id):
r = ctx["session"].delete(f"node/{node_id}")
exit_with(handle_request_error(r))
@node.command(name="create", aliases=["c"])
@click.argument("name", type=click.STRING)
@click.argument("api_url", type=click.STRING)
@click.argument("access_key_id", type=click.STRING)
@click.argument("secret_access_key", type=click.STRING)
@click.option("--region-name", type=click.STRING, default="us-east-1")
@click.option("--sts-api-url", type=click.STRING)
@click.option(
"--role-arn",
type=click.STRING,
help="ARN for role to use during STS assume role. Required for s3, Ignored for MinIO. Make sure this role has NO permissions",
)
@click.pass_obj
def register(
ctx,
name,
api_url,
access_key_id,
secret_access_key,
region_name,
sts_api_url,
role_arn,
):
r = ctx["session"].post(
"node",
json={
"name": name,
"api_url": api_url,
"access_key_id": access_key_id,
"secret_access_key": secret_access_key,
"region_name": region_name,
"sts_api_url": sts_api_url,
"assume_role_arn": role_arn,
},
)
exit_with(handle_request_error(r))
cli.add_command(node)
def register_to(self, group: click.Group) -> None:
for command in self.commands:
group.add_command(command)
def db_commands(group: click.Group) -> click.Group:
group.add_command(create_all)
group.add_command(drop_all)
return group
def add_commands(command_group: click.Group) -> None:
command_group.add_command(start)
command_group.add_command(stop)
command_group.add_command(restart)
command_group.add_command(reboot)
command_group.add_command(init)
# command_group.add_command(settheme)
command_group.add_command(dc_command)
command_group.add_command(run)
command_group.add_command(bindmount_command)
command_group.add_command(execute)
command_group.add_command(logs)
"""
Command Line Interface
"""
import sys
from click import Group, option, argument
cli = Group(help="""Welcome to Progames""")
game = Group()
cli.add_command(game, "game")
@cli.command()
def start():
"""Start server"""
print("Server started")
@game.command("install")
@argument("uri")
def _import(uri):
"""Import game"""
print("install game from:", uri)
def main():
cli.main(args=sys.argv[1:])
def add_commands(cli: click.Group) -> None:
cli.add_command(lint)
def vdk_command_line(root_command: click.Group) -> None:
root_command.add_command(ingest_csv)
def make(cli: click.Group):
@cli.group(name="root", cls=ClickAliasedGroup, aliases=["r"])
def root():
pass
@root.command(name="list", aliases=["l", "ls"])
@click.option("--node-name", type=click.STRING)
@click.pass_obj
def list_roots(ctx, node_name):
r = ctx["session"].get("root", params={"node_name": node_name})
exit_with(handle_request_error(r))
@root.command(name="create", aliases=["c"])
@click.argument("bucket", type=click.STRING)
@click.argument("node_name", type=click.STRING)
@click.option("--base-path", type=click.STRING, default="")
@click.option(
"--root-type",
type=click.Choice(schemas.RootType),
default=schemas.RootType.PRIVATE.value,
)
@click.pass_obj
def create_root(ctx, bucket, node_name, base_path, root_type):
r = ctx["session"].post(
"root",
json={
"bucket": bucket,
"node_name": node_name,
"base_path": base_path,
"root_type": root_type,
},
)
exit_with(handle_request_error(r))
@root.command(name="delete", aliases=["d"])
@click.argument("root_id", type=click.STRING)
@click.pass_obj
def delete_root(ctx, root_id):
r = ctx["session"].delete(f"root/{root_id}")
exit_with(handle_request_error(r))
@root.command(name="import", help="Import all workspaces in a root.")
@click.argument("root_id")
@click.option("--index-all", is_flag=True)
@click.pass_obj
def import_all_workspaces(ctx, root_id, index_all):
# Dynamic, expensive imports
from workspacesio.common import producers
ctx = config.getctx(ctx)
r = ctx.session.post(f"root/{root_id}/import")
if not r.ok:
exit_with(handle_request_error(r))
rdata = schemas.RootCredentials(**r.json())
root_contents = producers.minio_list_root_children(node=rdata.node,
root=rdata.root)
workspace_list: List[schemas.WorkspaceDB] = []
for folder in root_contents:
prefix = folder.object_name.lstrip(rdata.root.base_path).strip("/")
if len(prefix) > 0:
print(f"Discovered {prefix}")
workspace = ctx.session.post(
"workspace",
json={
"name": prefix,
"public": False,
"unmanaged": True,
"base_path": prefix,
"node_name": rdata.node.name,
"root_id": str(rdata.root.id),
},
)
if not workspace.ok and workspace.status_code != 409:
exit_with(handle_request_error(workspace))
workspace_list.append(schemas.WorkspaceDB(**workspace.json()))
@root.command(name="import-workspace",
help="Import a particular prefix as a workspace.")
@click.argument("root_id", type=click.STRING)
@click.argument("--base-path", type=click.STRING, default="")
def import_workspace(ctx, root_id, base_path):
pass
cli.add_command(root)
def pytask_extend_command_line_interface(cli: click.Group) -> None:
"""Extend the command line interface."""
cli.add_command(clean)
def __init__(
self,
cli: click.Group,
pipeline_func: Callable,
services: PlatformServicesBase,
image_url: str,
package_entrypoint: str,
op_builders: List[Callable[[HmlContainerOp], HmlContainerOp]],
envs: Dict[str, str],
):
if cli is None:
raise (TypeError("Parameter: `cli` must be supplied"))
if pipeline_func is None:
raise (TypeError("Parameter: `pipeline_func` must be supplied"))
if services is None:
raise (TypeError("Parameter: `services` must be supplied"))
if image_url is None or image_url == "":
raise (TypeError("Parameter: `image_url` must be supplied"))
if package_entrypoint is None or package_entrypoint == "":
raise (
TypeError("Parameter: `package_entrypoint` must be supplied"))
self.name = pipeline_func.__name__
self.envs: Dict[str, str] = envs
self.services = services
self.pipeline_func = pipeline_func
self.kubeflow_pipeline = dsl.pipeline(pipeline_func,
pipeline_func.__name__)
self.is_deploying = False
self.cron: Optional[str] = None
self.experiment: Optional[str] = None
self.image_url = image_url
self.package_entrypoint = package_entrypoint
# The methods we use to configure our Ops for running in Kubeflow
self.op_builders = op_builders
self.ops_list: List[HmlContainerOp] = []
self.ops_dict: Dict[str, HmlContainerOp] = {}
# We treat the pipeline as a "group" of commands, rather than actually executing
# anything. We can then bind a
self.cli_pipeline = click.group(name=pipeline_func.__name__)(_pass)
# Register this with the root `pipeline` command
cli.add_command(self.cli_pipeline)
# Create a command to execute the whole pipeline
self.cli_all = click.command(name="run-all")(self.run_all)
self.deploy_dev = self._apply_deploy_options(
click.command(name="deploy-dev")(self._deploy_dev))
self.deploy_prod = self._apply_deploy_options(
click.command(name="deploy-prod")(self._deploy_prod))
self.cli_pipeline.add_command(self.cli_all)
self.cli_pipeline.add_command(self.deploy_dev)
self.cli_pipeline.add_command(self.deploy_prod)
def _register(self, parent: click.Group):
parent.add_command(self)
from click import Group from cli.start import start from cli.stop import stop from cli.logs import logs from cli.build import build from cli.kill import kill cli = Group() cli.add_command(start) cli.add_command(stop) cli.add_command(logs) cli.add_command(build) cli.add_command(kill)
def register(cli: click.Group):
cli.add_command(build)
cli.add_command(clean)
cli.add_command(pause)
cli.add_command(print_tokens)
cli.add_command(reset)
cli.add_command(resume)
cli.add_command(run_docker_command)
cli.add_command(scale)
cli.add_command(setup)
cli.add_command(start)
cli.add_command(validate)
cli.add_command(worker)
