def uninstall():
"""
Removes cloud resources, local config, and python package.
"""
click.secho('''DANGER WILL ROBINSON, This will:
- Destroy all nodes in the cloud
- Remove all docker images on your computer
- Delete the .numerai configuration directory on your computer
- Uninstall the numerai-cli python package
- Leave Python and Docker installed on your computer
''',
fg='red')
if not click.confirm('Are you absolutely sure you want to uninstall?'):
return
if os.path.exists(CONFIG_PATH):
if len(os.listdir(CONFIG_PATH)) == 0:
os.rmdir(CONFIG_PATH)
else:
napi = base_api.Api(*get_numerai_keys())
node_config = load_or_init_nodes()
click.secho('deregistering all webhooks...')
for node, config in node_config.items():
napi.set_submission_webhook(config['model_id'], None)
click.secho('destroying cloud resources...')
all_keys = load_or_init_keys()
provider_keys = {}
for provider in PROVIDERS:
if provider in all_keys.keys():
provider_keys.update(all_keys[provider])
terraform('destroy -auto-approve',
verbose=True,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
click.secho('cleaning up docker images...')
subprocess.run('docker system prune -f -a --volumes', shell=True)
shutil.rmtree(CONFIG_PATH)
click.secho('uninstalling python package...')
res = subprocess.run('pip3 uninstall numerai-cli -y',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if res.returncode != 0:
if b'PermissionError' in res.stderr:
click.secho(
'uninstall failed due to permissions, '
'run "pip3 uninstall numerai-cli -y" manually '
'to ensure the package was uninstalled',
fg='red')
else:
click.secho(f'Unexpected error occurred:\n{res.stderr}', fg='red')
click.secho("All those moments will be lost in time, like tears in rain.",
fg='red')
def destroy(ctx, verbose):
"""
Uses Terraform to destroy Numerai Compute cluster in AWS.
This will delete everything, including:
- lambda url
- docker container and associated task
- all logs
This command is idempotent and safe to run multiple times.
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
if not os.path.exists(CONFIG_PATH):
click.secho(f".numerai directory not setup, run `numerai setup`...",
fg='red')
return
try:
nodes_config = load_or_init_nodes()
node_config = nodes_config[node]
provider_keys = get_provider_keys(node)
except (KeyError, FileNotFoundError) as e:
click.secho(
f"make sure you run `numerai setup` and "
f"`numerai node -n {node} config` first...",
fg='red')
return
try:
click.secho(f"deleting node configuration...")
del nodes_config[node]
store_config(NODES_PATH, nodes_config)
click.secho(f"deleting cloud resources for node...")
terraform(f'apply -auto-approve',
verbose,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
except Exception as e:
click.secho(e.__str__(), fg='red')
nodes_config[node] = node_config
store_config(NODES_PATH, nodes_config)
return
if 'model_id' in node_config and 'webhook_url' in node_config:
napi = base_api.Api(*get_numerai_keys())
model_id = node_config['model_id']
webhook_url = node_config['webhook_url']
click.echo(
f'deregistering webhook {webhook_url} for model {model_id}...')
napi.set_submission_webhook(model_id, None)
click.secho("Prediction Node destroyed successfully", fg='green')
def node(ctx, model_name, signals):
"""
Commands to manage and test Prediction Nodes.
"""
if not os.path.exists(CONFIG_PATH):
click.secho(
'cannot find .numerai config directory, '
'run `numerai setup`',
fg='red')
exit(1)
if signals:
tournament = TOURNAMENT_SIGNALS
name_prefix = 'signals'
else:
tournament = TOURNAMENT_NUMERAI
name_prefix = 'numerai'
napi = base_api.Api(*get_numerai_keys())
models = napi.get_models(tournament)
try:
model_id = models[model_name]
ctx.ensure_object(dict)
ctx.obj['model'] = {
'id': model_id,
'name': f'{name_prefix}-{model_name}',
'is_signals': signals
}
except KeyError:
click.secho(
f'No tournament {tournament} model with name "{model_name}" '
f'found in list of models:\n{json.dumps(models, indent=2)}'
f'\n(use the "-s" flag for signals models)',
fg='red')
exit(1)
def config(ctx, verbose, provider, size, path, example, cron, register_webhook):
"""
Uses Terraform to create a full Numerai Compute cluster in AWS.
Prompts for your AWS and Numerai API keys on first run, caches them in $HOME/.numerai.
At the end of running, this will output a config file 'nodes.json'.
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
model_id = model['id']
if example is not None:
path = copy_example(example, path, verbose)
# get nodes config object and set defaults for this node
click.secho(f'configuring node "{node}"...')
nodes_config = load_or_init_nodes()
nodes_config.setdefault(node, {})
nodes_config[node].update({
key: default
for key, default in DEFAULT_SETTINGS.items()
if key not in nodes_config[node]
})
# update node as needed
node_conf = nodes_config[node]
if provider:
node_conf['provider'] = provider
if size:
node_conf['cpu'] = SIZE_PRESETS[size][0]
node_conf['memory'] = SIZE_PRESETS[size][1]
if path:
node_conf['path'] = os.path.abspath(path)
if model_id:
node_conf['model_id'] = model_id
if cron:
node_conf['cron'] = cron
nodes_config[node] = node_conf
# double check there is a dockerfile in the path we are about to configure
check_for_dockerfile(nodes_config[node]['path'])
store_config(NODES_PATH, nodes_config)
# terraform apply
provider_keys = get_provider_keys(node)
click.secho(f'running terraform to provision cloud infrastructure...')
terraform(f'apply -auto-approve', verbose,
env_vars=provider_keys,
inputs={'node_config_file': 'nodes.json'})
click.secho('cloud resources created successfully', fg='green')
# terraform output for AWS nodes
click.echo(f'saving node configuration to {NODES_PATH}...')
res = terraform(f"output -json aws_nodes", verbose).decode('utf-8')
try:
aws_nodes = json.loads(res)
except json.JSONDecodeError:
click.secho("failed to save node configuration, pleas retry.", fg='red')
return
for node_name, data in aws_nodes.items():
nodes_config[node_name].update(data)
store_config(NODES_PATH, nodes_config)
if verbose:
click.secho(f'new config:\n{json.dumps(load_or_init_nodes(), indent=2)}')
webhook_url = nodes_config[node]['webhook_url']
napi = base_api.Api(*get_numerai_keys())
if not cron or register_webhook:
click.echo(f'registering webhook {webhook_url} for model {model_id}...')
napi.set_submission_webhook(model_id, webhook_url)
else:
click.echo(f'removing registered webhook for model {model_id}...')
napi.set_submission_webhook(model_id, None)
click.secho('Prediction Node configured successfully. '
'Next: deploy and test your node', fg='green')
def api_fixture():
api = base_api.Api(verbosity='DEBUG')
return api
def test_NumerAPI():
# invalid log level should raise
with pytest.raises(AttributeError):
base_api.Api(verbosity="FOO")
def test(ctx, local, command, verbose):
"""
Full end-to-end cloud or local test for a Prediction Node.
This checks that:
1. Numerai can reach the Trigger
2. The Trigger schedules a Container to run
3. The Container starts up on the Compute Cluster
4. The Container uploads a submission with the Trigger ID assigned to it
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
is_signals = model['is_signals']
node_config = load_or_init_nodes(node)
if local:
click.secho("starting local test; building container...")
docker.build(node_config, verbose)
click.secho("running container...")
docker.run(node_config, verbose, command=command)
api = base_api.Api(*get_numerai_keys())
try:
click.secho("Checking if Numerai can Trigger your model...")
res = api.raw_query('''mutation ( $modelId: String! ) {
triggerModelWebhook( modelId: $modelId )
}''',
variables={
'modelId': node_config['model_id'],
},
authorization=True)
trigger_id = res['data']['triggerModelWebhook']
if verbose:
click.echo(f"response:\n{res}")
click.secho(f"Webhook reachable...", fg='green')
click.secho(f"Trigger ID assigned for this test: {trigger_id}",
fg='green')
except ValueError as e:
click.secho(f'there was a problem calling your webhook...', fg='red')
if 'Internal Server Error' in str(e):
click.secho('attempting to dump webhook logs', fg='red')
monitor(node, node_config, True, 20, LOG_TYPE_WEBHOOK, False)
return
click.secho("checking task status...")
monitor(node, node_config, verbose, 15, LOG_TYPE_CLUSTER, follow_tail=True)
click.secho("checking for submission...")
res = api.raw_query('''query ( $modelId: String! ) {
submissions( modelId: $modelId ){
round{ number, tournament },
triggerId
}
}''',
variables={'modelId': node_config['model_id']},
authorization=True)
tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI
round = api.get_current_round(tournament)
submission_triggers = [
sub['triggerId'] for sub in res['data']['submissions']
if sub['round']['number'] == round
]
if len(submission_triggers) == 0:
click.secho("No submission found for current round, test failed",
fg='red')
return
if trigger_id not in submission_triggers:
click.secho(
"Your node did not submit the Trigger ID assigned during this test, "
"please ensure your node uses numerapi >= 0.2.4 (ignore if using rlang)",
fg='red')
return
else:
click.secho("Submission uploaded correctly", fg='green')
click.secho("✓ Test complete, your model now submits automatically!",
fg='green')
def test(ctx, local, command, verbose):
"""
Full end-to-end cloud or local test for a Prediction Node.
This checks that:
1. Numerai can reach the Trigger
2. The Trigger schedules a Container to run
3. The Container starts up on the Compute Cluster
4. The Container uploads a submission with the Trigger ID assigned to it
"""
ctx.ensure_object(dict)
model = ctx.obj['model']
node = model['name']
is_signals = model['is_signals']
node_config = load_or_init_nodes(node)
if local:
click.secho("starting local test; building container...")
docker.build(node_config, verbose)
click.secho("running container...")
docker.run(node_config, verbose, command=command)
api = base_api.Api(*get_numerai_keys())
trigger_id = None
try:
if 'cron' in node_config:
click.secho("Attempting to manually trigger Cron node...")
res = requests.post(node_config['webhook_url'], json.dumps({}))
res.raise_for_status()
else:
click.secho("Checking if Numerai can Trigger your model...")
res = api.raw_query('''mutation ( $modelId: String! ) {
triggerModelWebhook( modelId: $modelId )
}''',
variables={
'modelId': node_config['model_id'],
},
authorization=True)
trigger_id = res['data']['triggerModelWebhook']
click.secho(f"Trigger ID assigned for this test: {trigger_id}",
fg='green')
if verbose:
click.echo(f"response:\n{res}")
click.secho(f"Webhook reachable...", fg='green')
except ValueError as e:
click.secho(f'there was a problem calling your webhook...', fg='red')
if 'Internal Server Error' in str(e):
click.secho('attempting to dump webhook logs', fg='red')
monitor(node, node_config, True, 20, LOG_TYPE_WEBHOOK, False)
return
click.secho("checking task status...")
monitor(node, node_config, verbose, 15, LOG_TYPE_CLUSTER, follow_tail=True)
click.secho("checking for submission...")
res = api.raw_query('''query ( $modelId: String! ) {
submissions( modelId: $modelId ){
round{ number, tournament },
triggerId
insertedAt
}
}''',
variables={'modelId': node_config['model_id']},
authorization=True)
tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI
curr_round = api.get_current_round(tournament)
latest_subs = sorted(filter(
lambda sub: sub['round']['number'] == curr_round,
res['data']['submissions']),
key=lambda sub: sub['insertedAt'],
reverse=True)
if len(latest_subs) == 0:
click.secho("No submission found for current round, test failed",
fg='red')
return
latest_sub = latest_subs[0]
if 'cron' in node_config:
latest_date = datetime.strptime(latest_sub['insertedAt'],
"%Y-%m-%dT%H:%M:%SZ")
if latest_date < datetime.utcnow() - timedelta(minutes=5):
click.secho(
"No submission appeared in the last 5 minutes, be sure that your node"
" is submitting correctly, check the numerai-cli wiki for more"
" information on how to monitor parts of your node.",
fg='red')
if trigger_id != latest_sub['triggerId']:
click.secho(
"Your node did not submit the Trigger ID assigned during this test, "
"please ensure your node uses numerapi >= 0.2.4 (ignore if using rlang)",
fg='red')
return
click.secho("Submission uploaded correctly", fg='green')
click.secho("Test complete, your model now submits automatically!",
fg='green')
