def uninstall():
"""
Removes cloud resources, local config, and python package.
"""
click.secho('''DANGER WILL ROBINSON, This will:
- Destroy all nodes in the cloud
- Remove all docker images on your computer
- Delete the .numerai configuration directory on your computer
- Uninstall the numerai-cli python package
- Leave Python and Docker installed on your computer
''',
fg='red')
if not click.confirm('Are you absolutely sure you want to uninstall?'):
return
if os.path.exists(CONFIG_PATH):
if len(os.listdir(CONFIG_PATH)) == 0:
os.rmdir(CONFIG_PATH)
else:
napi = base_api.Api(*get_numerai_keys())
node_config = load_or_init_nodes()
click.secho('deregistering all webhooks...')
for node, config in node_config.items():
napi.set_submission_webhook(config['model_id'], None)
click.secho('destroying cloud resources...')
all_keys = load_or_init_keys()
provider_keys = {}
for provider in PROVIDERS:
if provider in all_keys.keys():
provider_keys.update(all_keys[provider])
terraform('destroy -auto-approve',
verbose=True,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
click.secho('cleaning up docker images...')
subprocess.run('docker system prune -f -a --volumes', shell=True)
shutil.rmtree(CONFIG_PATH)
click.secho('uninstalling python package...')
res = subprocess.run('pip3 uninstall numerai-cli -y',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if res.returncode != 0:
if b'PermissionError' in res.stderr:
click.secho(
'uninstall failed due to permissions, '
'run "pip3 uninstall numerai-cli -y" manually '
'to ensure the package was uninstalled',
fg='red')
else:
click.secho(f'Unexpected error occurred:\n{res.stderr}', fg='red')
click.secho("All those moments will be lost in time, like tears in rain.",
fg='red')
def destroy(ctx, verbose):
"""
Uses Terraform to destroy Numerai Compute cluster in AWS.
This will delete everything, including:
- lambda url
- docker container and associated task
- all logs
This command is idempotent and safe to run multiple times.
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
if not os.path.exists(CONFIG_PATH):
click.secho(f".numerai directory not setup, run `numerai setup`...",
fg='red')
return
try:
nodes_config = load_or_init_nodes()
node_config = nodes_config[node]
provider_keys = get_provider_keys(node)
except (KeyError, FileNotFoundError) as e:
click.secho(
f"make sure you run `numerai setup` and "
f"`numerai node -n {node} config` first...",
fg='red')
return
try:
click.secho(f"deleting node configuration...")
del nodes_config[node]
store_config(NODES_PATH, nodes_config)
click.secho(f"deleting cloud resources for node...")
terraform(f'apply -auto-approve',
verbose,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
except Exception as e:
click.secho(e.__str__(), fg='red')
nodes_config[node] = node_config
store_config(NODES_PATH, nodes_config)
return
if 'model_id' in node_config and 'webhook_url' in node_config:
napi = base_api.Api(*get_numerai_keys())
model_id = node_config['model_id']
webhook_url = node_config['webhook_url']
click.echo(
f'deregistering webhook {webhook_url} for model {model_id}...')
napi.set_submission_webhook(model_id, None)
click.secho("Prediction Node destroyed successfully", fg='green')
def setup(provider, verbose):
"""
Initializes cli and provider API keys.
"""
# check for old format, tell user to run numerai upgrade first
if os.path.isfile(CONFIG_PATH) or os.path.isdir('.numerai'):
click.secho(
'It looks like you have an old configuration of numerai-cli,'
'run `numerai upgrade` first.')
return
# setup numerai keys
click.secho(
"Initializing numerai keys "
"(press enter to keep value in brackets)...",
fg='yellow')
maybe_create(KEYS_PATH, protected=True)
config_numerai_keys()
# setup provider keys
click.secho(
f"\nInitializing {provider} keys "
f"(press enter to keep value in brackets)...",
fg='yellow')
config_provider_keys(provider)
# copy tf files
click.secho("copying terraform files...")
copy_files(TERRAFORM_PATH, CONFIG_PATH, force=True, verbose=True)
# terraform init
click.secho("initializing terraform to provision cloud infrastructure...")
terraform("init -upgrade", verbose)
click.secho("Numerai API Keys setup and working", fg='green')
click.secho(f"{provider} API Keys setup and working", fg='green')
click.secho(f"Terraform files copied to {CONFIG_PATH}", fg='green')
click.echo('succesfully initialized numerai-cli')
def upgrade(verbose):
"""
Upgrades configuration from <=0.2 format to >=0.3 format.
"""
home = str(Path.home())
old_key_path = os.path.join(home, '.numerai')
old_config_path = os.path.join(os.getcwd(), '.numerai/')
if str(old_key_path) == str(old_config_path):
click.secho('You cannot run this command from your home directory.')
return
if not os.path.exists(old_config_path):
click.secho(
'Run this command from the directory in which you first ran `numerai setup`'
' for CLI version 0.1 and 0.2 (it should have the .numerai folder in it)'
)
return
click.secho(
f"Upgrading, do not interrupt or else "
f"your environment may be corrupted.",
fg='yellow')
# MOVE KEYS FILE
if os.path.isfile(old_key_path):
temp_key_path = os.path.join(old_config_path, '.keys')
click.secho(
f"\tmoving old keyfile from '{old_key_path}' to '{temp_key_path}'",
)
shutil.move(old_key_path, temp_key_path)
# MOVE CONFIG FILE
if os.path.exists(old_config_path):
click.secho(
f"\tmoving old config from {old_config_path} to {CONFIG_PATH}", )
shutil.move(old_config_path, CONFIG_PATH)
# INIT KEYS AND NODES
keys_config = load_or_init_keys()
if not os.path.exists(
KEYS_PATH
) or 'aws' not in keys_config or 'numerai' not in keys_config:
click.secho(
f"Keys missing from {KEYS_PATH}, you must re-initialize your keys:"
)
config_numerai_keys()
config_provider_keys(PROVIDER_AWS)
nodes_config = load_or_init_nodes()
# DELETE OLD CONFIG FILES
click.secho('Checking for old config output files...', fg='yellow')
old_suburl_path = os.path.join(CONFIG_PATH, 'submission_url.txt')
if os.path.exists(old_suburl_path):
click.secho(f"\tdeleting {old_suburl_path}, you can populate the "
f"new nodes.json file with `numerai node config`")
os.remove(old_suburl_path)
old_docker_path = os.path.join(CONFIG_PATH, 'docker_repo.txt')
if os.path.exists(old_docker_path):
click.secho(f"\tdeleting {old_docker_path}, you can populate the "
f"new nodes.json file with `numerai node config`")
os.remove(old_docker_path)
# UPGRADE, RENAME, AND UPDATE TERRAFORM FILES
click.secho('Upgrading terraform files...', fg='yellow')
try:
with open(os.path.join(CONFIG_PATH, 'terraform.tfstate')) as f:
tfstate = json.load(f)
keys_config = load_or_init_keys('aws')
if '0.12' in tfstate['terraform_version']:
terraform('0.13upgrade -yes',
verbose,
version='0.13.6',
env_vars=keys_config)
terraform('init', verbose, version='0.13.6', env_vars=keys_config)
terraform('apply -auto-approve',
verbose,
version='0.13.6',
env_vars=keys_config)
except FileNotFoundError:
pass
except click.ClickException:
click.secho("Failed to upgrade to terraform state!", fg='red')
return
except Exception as e:
click.secho(f'Uncaught exception: {str(e)}', fg='red')
return
tf_files_map = {
'main.tf': '-main.tf',
'variables.tf': '-inputs.tf',
'outputs.tf': '-outputs.tf'
}
for old_file, new_file in tf_files_map.items():
old_file = os.path.join(CONFIG_PATH, old_file)
new_file = os.path.join(CONFIG_PATH, new_file)
if not os.path.exists(old_file):
continue
if not os.path.exists(old_file):
click.secho(
f'\trenaming {old_file} to {new_file} to prep for upgrade...')
shutil.move(old_file, new_file)
else:
os.remove(old_file)
copy_files(TERRAFORM_PATH, CONFIG_PATH, force=True, verbose=verbose)
# terraform init
click.secho("Re-initializing terraform...", fg='yellow')
terraform("init -upgrade", verbose=verbose)
if click.confirm(
"It's recommended you destroy your current Compute Node. Continue?"
):
click.secho("Removing old cloud infrastructure...", fg='yellow')
terraform('destroy -auto-approve',
verbose,
env_vars=load_or_init_keys('aws'),
inputs={'node_config_file': 'nodes.json'})
click.secho('Upgrade complete!', fg='green')
click.secho('run `numerai node config --help` to learn how to '
'register this directory as a prediction node')
def config(ctx, verbose, provider, size, path, example, cron, register_webhook):
"""
Uses Terraform to create a full Numerai Compute cluster in AWS.
Prompts for your AWS and Numerai API keys on first run, caches them in $HOME/.numerai.
At the end of running, this will output a config file 'nodes.json'.
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
model_id = model['id']
if example is not None:
path = copy_example(example, path, verbose)
# get nodes config object and set defaults for this node
click.secho(f'configuring node "{node}"...')
nodes_config = load_or_init_nodes()
nodes_config.setdefault(node, {})
nodes_config[node].update({
key: default
for key, default in DEFAULT_SETTINGS.items()
if key not in nodes_config[node]
})
# update node as needed
node_conf = nodes_config[node]
if provider:
node_conf['provider'] = provider
if size:
node_conf['cpu'] = SIZE_PRESETS[size][0]
node_conf['memory'] = SIZE_PRESETS[size][1]
if path:
node_conf['path'] = os.path.abspath(path)
if model_id:
node_conf['model_id'] = model_id
if cron:
node_conf['cron'] = cron
nodes_config[node] = node_conf
# double check there is a dockerfile in the path we are about to configure
check_for_dockerfile(nodes_config[node]['path'])
store_config(NODES_PATH, nodes_config)
# terraform apply
provider_keys = get_provider_keys(node)
click.secho(f'running terraform to provision cloud infrastructure...')
terraform(f'apply -auto-approve', verbose,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
click.secho('cloud resources created successfully', fg='green')
# terraform output for AWS nodes
click.echo(f'saving node configuration to {NODES_PATH}...')
res = terraform(f"output -json aws_nodes", verbose).decode('utf-8')
try:
aws_nodes = json.loads(res)
except json.JSONDecodeError:
click.secho("failed to save node configuration, pleas retry.", fg='red')
return
for node_name, data in aws_nodes.items():
nodes_config[node_name].update(data)
store_config(NODES_PATH, nodes_config)
if verbose:
click.secho(f'new config:\n{json.dumps(load_or_init_nodes(), indent=2)}')
webhook_url = nodes_config[node]['webhook_url']
napi = base_api.Api(*get_numerai_keys())
if not cron or register_webhook:
click.echo(f'registering webhook {webhook_url} for model {model_id}...')
napi.set_submission_webhook(model_id, webhook_url)
else:
click.echo(f'removing registered webhook for model {model_id}...')
napi.set_submission_webhook(model_id, None)
click.secho('Prediction Node configured successfully. '
'Next: deploy and test your node', fg='green')