def destroy(ctx, verbose): """ Uses Terraform to destroy Numerai Compute cluster in AWS. This will delete everything, including: - lambda url - docker container and associated task - all logs This command is idempotent and safe to run multiple times. """ ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] if not os.path.exists(CONFIG_PATH): click.secho(f".numerai directory not setup, run `numerai setup`...", fg='red') return try: nodes_config = load_or_init_nodes() node_config = nodes_config[node] provider_keys = get_provider_keys(node) except (KeyError, FileNotFoundError) as e: click.secho( f"make sure you run `numerai setup` and " f"`numerai node -n {node} config` first...", fg='red') return try: click.secho(f"deleting node configuration...") del nodes_config[node] store_config(NODES_PATH, nodes_config) click.secho(f"deleting cloud resources for node...") terraform(f'apply -auto-approve', verbose, env_vars=provider_keys, inputs={'node_config_file': 'nodes.json'}) except Exception as e: click.secho(e.__str__(), fg='red') nodes_config[node] = node_config store_config(NODES_PATH, nodes_config) return if 'model_id' in node_config and 'webhook_url' in node_config: napi = base_api.Api(*get_numerai_keys()) model_id = node_config['model_id'] webhook_url = node_config['webhook_url'] click.echo( f'deregistering webhook {webhook_url} for model {model_id}...') napi.set_submission_webhook(model_id, None) click.secho("Prediction Node destroyed successfully", fg='green')
def status(ctx, verbose, num_lines, log_type, follow_tail): """ Get the logs from the latest task for this node Logs are not created until a task is in the RUNNING state, so the logs returned by this command might be out of date. """ ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] monitor(node, load_or_init_nodes(node), verbose, num_lines, log_type, follow_tail)
def deploy(ctx, verbose): """Builds and pushes your docker image to the AWS ECR repo""" ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] node_config = files.load_or_init_nodes(node) docker.check_for_dockerfile(node_config['path']) click.echo('building container image (this may take several minutes)...') docker.build(node_config, verbose) click.echo('logging into container registry...') docker.login(node_config, verbose) click.echo('pushing image to registry (this may take several minutes)...') docker.push(node_config['docker_repo'], verbose) click.echo('cleaning up local images...') docker.cleanup(node_config) click.secho('Prediction Node deployed. Next: test your node.', fg='green')
def get_provider_keys(node): provider = load_or_init_nodes(node)['provider'] return load_or_init_keys(provider)
def doctor(): """ Checks and repairs your environment in case of errors. Attempts to provide information to debug your local machine. """ # Check environment pre-reqs click.secho("Running the environment setup script for your OS...") env_setup_cmd = None env_setup_status = -1 env_setup_err = '' if sys.platform == "linux" or sys.platform == "linux2": env_setup_cmd = 'sudo apt update && sudo apt install -y libcurl4 curl && ' \ 'sudo curl https://raw.githubusercontent.com/numerai/numerai-cli/master/scripts/setup-ubu.sh ' \ '| sudo bash' elif sys.platform == "darwin": env_setup_cmd = 'curl https://raw.githubusercontent.com/numerai/numerai-cli/master/scripts/setup-mac.sh | bash' elif is_win10(): env_setup_cmd = 'powershell -command "$Script = Invoke-WebRequest ' \ '\'https://raw.githubusercontent.com/numerai/numerai-cli/master/scripts/setup-win10.ps1\'; ' \ '$ScriptBlock = [ScriptBlock]::Create($Script.Content); Invoke-Command -ScriptBlock $ScriptBlock"' elif is_win8(): # TODO: check if more is needed? env_setup_cmd = 'docker info' else: env_setup_status = 1 env_setup_err = f"Unrecognized Operating System {sys.platform}, " \ f"cannot run environment setup script, skipping..." if env_setup_cmd is not None: res = subprocess.run(env_setup_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) env_setup_status = res.returncode env_setup_err = res.stderr # Check official (non-dev) version click.secho(f"Checking your numerai-cli version...") res = str( subprocess.run('pip3 show numerai-cli', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) curr_ver = [s for s in res.split('\\n') if 'Version:' in s][0].split(': ')[1] url = f"https://pypi.org/pypi/numerai-cli/json" versions = list( reversed( sorted( filter(lambda key: 'dev' not in key, json.load(request.urlopen(url))["releases"].keys())))) # Check keys click.secho("Checking your API keys...") nodes_config = load_or_init_nodes() used_providers = [nodes_config[n]['provider'] for n in nodes_config] invalid_providers = [] try: check_numerai_validity(*get_numerai_keys()) except: invalid_providers.append('numerai') if 'aws' in used_providers: try: check_aws_validity(*get_aws_keys()) except: invalid_providers.append('aws') if env_setup_status != 0: click.secho(f"✖ Environment setup incomplete:", fg='red') click.secho(env_setup_err, fg='red') click.secho( f"Ensure your OS is supported and read the Troubleshooting wiki: " f"https://github.com/numerai/numerai-cli/wiki/Troubleshooting", fg='red') else: click.secho("✓ Environment setup with Docker and Python", fg='green') if curr_ver < versions[0]: click.secho( f"✖ numerai-cli needs an upgrade" f"(run `pip3 install -U numerai-cli` to fix)", fg='red') else: click.secho("✓ numerai-cli is up to date", fg='green') if len(invalid_providers): click.secho( f"✖ Invalid provider keys: {invalid_providers}" f"(run `numerai setup` to fix)", fg='red') else: click.secho("✓ API Keys working", fg='green') click.secho( "\nIf you need help troubleshooting or want to report a bug please read the" "\nTroubleshooting and Feedback section of the readme:" "\nhttps://github.com/numerai/numerai-cli#troubleshooting-and-feedback", fg='yellow')
def config(ctx, verbose, provider, size, path, example, cron, register_webhook): """ Uses Terraform to create a full Numerai Compute cluster in AWS. Prompts for your AWS and Numerai API keys on first run, caches them in $HOME/.numerai. At the end of running, this will output a config file 'nodes.json'. """ ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] model_id = model['id'] if example is not None: path = copy_example(example, path, verbose) # get nodes config object and set defaults for this node click.secho(f'configuring node "{node}"...') nodes_config = load_or_init_nodes() nodes_config.setdefault(node, {}) nodes_config[node].update({ key: default for key, default in DEFAULT_SETTINGS.items() if key not in nodes_config[node] }) # update node as needed node_conf = nodes_config[node] if provider: node_conf['provider'] = provider if size: node_conf['cpu'] = SIZE_PRESETS[size][0] node_conf['memory'] = SIZE_PRESETS[size][1] if path: node_conf['path'] = os.path.abspath(path) if model_id: node_conf['model_id'] = model_id if cron: node_conf['cron'] = cron nodes_config[node] = node_conf # double check there is a dockerfile in the path we are about to configure check_for_dockerfile(nodes_config[node]['path']) store_config(NODES_PATH, nodes_config) # terraform apply provider_keys = get_provider_keys(node) click.secho(f'running terraform to provision cloud infrastructure...') terraform(f'apply -auto-approve', verbose, env_vars=provider_keys, inputs={'node_config_file': 'nodes.json'}) click.secho('cloud resources created successfully', fg='green') # terraform output for AWS nodes click.echo(f'saving node configuration to {NODES_PATH}...') res = terraform(f"output -json aws_nodes", verbose).decode('utf-8') try: aws_nodes = json.loads(res) except json.JSONDecodeError: click.secho("failed to save node configuration, pleas retry.", fg='red') return for node_name, data in aws_nodes.items(): nodes_config[node_name].update(data) store_config(NODES_PATH, nodes_config) if verbose: click.secho(f'new config:\n{json.dumps(load_or_init_nodes(), indent=2)}') webhook_url = nodes_config[node]['webhook_url'] napi = base_api.Api(*get_numerai_keys()) if not cron or register_webhook: click.echo(f'registering webhook {webhook_url} for model {model_id}...') napi.set_submission_webhook(model_id, webhook_url) else: click.echo(f'removing registered webhook for model {model_id}...') napi.set_submission_webhook(model_id, None) click.secho('Prediction Node configured successfully. ' 'Next: deploy and test your node', fg='green')
def test(ctx, local, command, verbose): """ Full end-to-end cloud or local test for a Prediction Node. This checks that: 1. Numerai can reach the Trigger 2. The Trigger schedules a Container to run 3. The Container starts up on the Compute Cluster 4. The Container uploads a submission with the Trigger ID assigned to it """ ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] is_signals = model['is_signals'] node_config = load_or_init_nodes(node) if local: click.secho("starting local test; building container...") docker.build(node_config, verbose) click.secho("running container...") docker.run(node_config, verbose, command=command) api = base_api.Api(*get_numerai_keys()) try: click.secho("Checking if Numerai can Trigger your model...") res = api.raw_query('''mutation ( $modelId: String! ) { triggerModelWebhook( modelId: $modelId ) }''', variables={ 'modelId': node_config['model_id'], }, authorization=True) trigger_id = res['data']['triggerModelWebhook'] if verbose: click.echo(f"response:\n{res}") click.secho(f"Webhook reachable...", fg='green') click.secho(f"Trigger ID assigned for this test: {trigger_id}", fg='green') except ValueError as e: click.secho(f'there was a problem calling your webhook...', fg='red') if 'Internal Server Error' in str(e): click.secho('attempting to dump webhook logs', fg='red') monitor(node, node_config, True, 20, LOG_TYPE_WEBHOOK, False) return click.secho("checking task status...") monitor(node, node_config, verbose, 15, LOG_TYPE_CLUSTER, follow_tail=True) click.secho("checking for submission...") res = api.raw_query('''query ( $modelId: String! ) { submissions( modelId: $modelId ){ round{ number, tournament }, triggerId } }''', variables={'modelId': node_config['model_id']}, authorization=True) tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI round = api.get_current_round(tournament) submission_triggers = [ sub['triggerId'] for sub in res['data']['submissions'] if sub['round']['number'] == round ] if len(submission_triggers) == 0: click.secho("No submission found for current round, test failed", fg='red') return if trigger_id not in submission_triggers: click.secho( "Your node did not submit the Trigger ID assigned during this test, " "please ensure your node uses numerapi >= 0.2.4 (ignore if using rlang)", fg='red') return else: click.secho("Submission uploaded correctly", fg='green') click.secho("✓ Test complete, your model now submits automatically!", fg='green')
def test(ctx, local, command, verbose): """ Full end-to-end cloud or local test for a Prediction Node. This checks that: 1. Numerai can reach the Trigger 2. The Trigger schedules a Container to run 3. The Container starts up on the Compute Cluster 4. The Container uploads a submission with the Trigger ID assigned to it """ ctx.ensure_object(dict) model = ctx.obj['model'] node = model['name'] is_signals = model['is_signals'] node_config = load_or_init_nodes(node) if local: click.secho("starting local test; building container...") docker.build(node_config, verbose) click.secho("running container...") docker.run(node_config, verbose, command=command) api = base_api.Api(*get_numerai_keys()) trigger_id = None try: if 'cron' in node_config: click.secho("Attempting to manually trigger Cron node...") res = requests.post(node_config['webhook_url'], json.dumps({})) res.raise_for_status() else: click.secho("Checking if Numerai can Trigger your model...") res = api.raw_query('''mutation ( $modelId: String! ) { triggerModelWebhook( modelId: $modelId ) }''', variables={ 'modelId': node_config['model_id'], }, authorization=True) trigger_id = res['data']['triggerModelWebhook'] click.secho(f"Trigger ID assigned for this test: {trigger_id}", fg='green') if verbose: click.echo(f"response:\n{res}") click.secho(f"Webhook reachable...", fg='green') except ValueError as e: click.secho(f'there was a problem calling your webhook...', fg='red') if 'Internal Server Error' in str(e): click.secho('attempting to dump webhook logs', fg='red') monitor(node, node_config, True, 20, LOG_TYPE_WEBHOOK, False) return click.secho("checking task status...") monitor(node, node_config, verbose, 15, LOG_TYPE_CLUSTER, follow_tail=True) click.secho("checking for submission...") res = api.raw_query('''query ( $modelId: String! ) { submissions( modelId: $modelId ){ round{ number, tournament }, triggerId insertedAt } }''', variables={'modelId': node_config['model_id']}, authorization=True) tournament = TOURNAMENT_SIGNALS if is_signals else TOURNAMENT_NUMERAI curr_round = api.get_current_round(tournament) latest_subs = sorted(filter( lambda sub: sub['round']['number'] == curr_round, res['data']['submissions']), key=lambda sub: sub['insertedAt'], reverse=True) if len(latest_subs) == 0: click.secho("No submission found for current round, test failed", fg='red') return latest_sub = latest_subs[0] if 'cron' in node_config: latest_date = datetime.strptime(latest_sub['insertedAt'], "%Y-%m-%dT%H:%M:%SZ") if latest_date < datetime.utcnow() - timedelta(minutes=5): click.secho( "No submission appeared in the last 5 minutes, be sure that your node" " is submitting correctly, check the numerai-cli wiki for more" " information on how to monitor parts of your node.", fg='red') if trigger_id != latest_sub['triggerId']: click.secho( "Your node did not submit the Trigger ID assigned during this test, " "please ensure your node uses numerapi >= 0.2.4 (ignore if using rlang)", fg='red') return click.secho("Submission uploaded correctly", fg='green') click.secho("Test complete, your model now submits automatically!", fg='green')