def download_binary(net, uname):
commit = latest_deployed_version(net)
branch = latest_deployed_release(net)
if commit:
print(f'Downloading latest deployed version for {net}')
download_near_s3(f'nearcore/{uname}/{branch}/{commit}/near',
os.path.expanduser(f'~/.nearup/near/{net}/near'))
download_near_s3(
f'nearcore/{uname}/{branch}/{commit}/keypair-generator',
os.path.expanduser(f'~/.nearup/near/{net}/keypair-generator'))
download_near_s3(
f'nearcore/{uname}/{branch}/{commit}/genesis-csv-to-json',
os.path.expanduser(f'~/.nearup/near/{net}/genesis-csv-to-json'))
subprocess.check_output(
['chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/near')])
subprocess.check_output([
'chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/keypair-generator')
])
subprocess.check_output([
'chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/genesis-csv-to-json')
])
with open(os.path.expanduser(f'~/.nearup/near/{net}/version'),
'w') as f:
f.write(commit)
def show_logs(follow):
LOGS_FOLDER = os.path.expanduser('~/.nearup/logs')
if not os.path.exists(NODE_PID):
print('Node is not running')
exit(1)
pid_info = open(NODE_PID).read()
if 'devnet' in pid_info:
net = 'devnet'
elif 'betanet' in pid_info:
net = 'betanet'
elif 'testnet' in pid_info:
net = 'testnet'
else:
# TODO: localnet could have several logs, not showing them all but list log files here
# Maybe better to support `nearup logs node0` usage.
print(
f'You are running local net. Logs are in: ~/.nearup/localnet-logs/'
)
exit(0)
command = ['tail']
if follow:
command += ['-f']
command += [os.path.expanduser(f'~/.nearup/logs/{net}.log')]
try:
subprocess.run(command, start_new_session=True)
except KeyboardInterrupt:
exit(0)
def download_binary(net, uname):
latest_deploy_version = binary_changed(net)
if latest_deploy_version:
print(f'Downloading latest deployed version for {net}')
download_near_s3(
f'nearcore/{uname}/{net_to_branch(net)}/{latest_deploy_version}/near',
os.path.expanduser(f'~/.nearup/near/{net}/near'))
download_near_s3(
f'nearcore/{uname}/{net_to_branch(net)}/{latest_deploy_version}/keypair-generator',
os.path.expanduser(f'~/.nearup/near/{net}/keypair-generator'))
download_near_s3(
f'nearcore/{uname}/{net_to_branch(net)}/{latest_deploy_version}/genesis-csv-to-json',
os.path.expanduser(f'~/.nearup/near/{net}/genesis-csv-to-json'))
subprocess.check_output(
['chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/near')])
subprocess.check_output([
'chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/keypair-generator')
])
subprocess.check_output([
'chmod', '+x',
os.path.expanduser(f'~/.nearup/near/{net}/genesis-csv-to-json')
])
with open(os.path.expanduser(f'~/.nearup/near/{net}/version'),
'w') as f:
f.write(latest_deploy_version)
def check_binary_version(binary_path, chain_id):
latest_deploy_version = latest_deployed_version(chain_id)
version = subprocess.check_output(
[f'{binary_path}/near', '--version'],
universal_newlines=True).split('(build ')[1].split(')')[0]
if not latest_deploy_version.startswith(version):
print(
f'Warning: current deployed version on {chain_id} is {latest_deploy_version}, but local binary is {version}. It might not work'
)
def check_and_update_genesis(chain_id, home_dir):
if genesis_changed(chain_id, home_dir):
print(f'Update genesis config and remove stale data for {chain_id}')
os.remove(os.path.join(home_dir, 'genesis.json'))
download_genesis(chain_id, home_dir)
if os.path.exists(os.path.join(home_dir, 'data')):
shutil.rmtree(os.path.join(home_dir, 'data'))
return True
return False
def binary_changed(net):
latest_deploy_version = latest_deployed_version(net)
if os.path.exists(os.path.expanduser(f'~/.nearup/near/{net}/version')):
with open(os.path.expanduser(f'~/.nearup/near/{net}/version')) as f:
version = f.read().strip()
if version == latest_deploy_version:
print('Downloaded binary version is up to date')
return False
return latest_deploy_version
def print_staking_key(home_dir):
key_path = os.path.join(home_dir, 'validator_key.json')
if not os.path.exists(key_path):
return
key_file = json.loads(open(key_path).read())
if not key_file['account_id']:
print("Node is not staking. Re-run init to specify staking account.")
return
print("Stake for user '%s' with '%s'" %
(key_file['account_id'], key_file['public_key']))
def genesis_changed(chain_id, home_dir):
genesis_md5sum = get_genesis_md5sum(chain_id)
local_genesis_md5sum = hashlib.md5(
open(os.path.join(os.path.join(home_dir, 'genesis.json')),
'rb').read()).hexdigest()
if genesis_md5sum == local_genesis_md5sum:
print(f'Our genesis version is up to date')
return False
else:
print(
f'Remote genesis protocol version md5 {genesis_md5sum}, ours is {local_genesis_md5sum}'
)
return True
def docker_changed(net):
image = net_to_docker_image(net)
print(f'Docker image: {image}')
local_version = subprocess.check_output(
['docker', 'image', 'ls', '-q', '--filter', f'reference={image}'],
universal_newlines=True).strip()
if local_version:
print(f'Local docker version: {local_version}')
repo, *tag = image.split(':')
if tag:
tag = tag[0]
else:
tag = 'latest'
auth_token = json.loads(
download(
f"https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull"
))['token']
image_info = json.loads(
download(
f"https://index.docker.io/v2/{repo}/manifests/{tag}",
headers=[
f"Authorization: Bearer {auth_token}",
"Accept: application/vnd.docker.distribution.manifest.v2+json"
]))
remote_version = image_info["config"]["digest"].split(':')[1]
print(f'Remote version: {remote_version}')
if remote_version.startswith(local_version):
print('Local docker image is up to date')
return False
return True
def create_genesis(home, binary_path, nodocker, image, chain_id,
tracked_shards):
if os.path.exists(os.path.join(home, 'genesis.json')):
print("Genesis already exists")
return
print("Creating genesis...")
if not os.path.exists(os.path.join(home, 'accounts.csv')):
raise Exception(
"Failed to generate genesis: accounts.csv does not exist")
if nodocker:
cmd = [f'{binary_path}/genesis-csv-to-json']
cmd.extend(['--home', home])
cmd.extend(['--chain-id', chain_id])
if len(tracked_shards) > 0:
cmd.extend(['--tracked-shards', tracked_shards])
try:
subprocess.call(cmd)
except KeyboardInterrupt:
print("\nStopping NEARCore.")
else:
subprocess.check_output(['mkdir', '-p', home])
subprocess.check_output([
'docker', 'run', '-u', USER, '-v',
'%s:/srv/genesis-csv-to-json' % home, image, 'genesis-csv-to-json',
'--home=/srv/genesis-csv-to-json',
'--chain-id=%s' % chain_id,
'--tracked-shards=%s' % tracked_shards
])
print("Genesis created")
def start_stakewars(home, binary_path, nodocker, image, verbose,
tracked_shards):
create_genesis(home, binary_path, nodocker, image, 'stakewars',
tracked_shards)
if nodocker:
run_nodocker(home,
binary_path,
boot_nodes='',
verbose=verbose,
chain_id='stakewars')
else:
run_docker(image, home, boot_nodes='', verbose=verbose)
print(
"Node is running! \nTo check logs call: docker logs --follow nearcore"
)
def check_exist_neard():
if os.path.exists(NODE_PID):
print("There is already binary nodes running. Stop it using:")
print("nearup stop")
print(f"If this is a mistake, remove {NODE_PID}")
exit(1)
if shutil.which('docker') is not None:
p = subprocess.Popen(['docker', 'ps', '-q', '-f', 'name=nearcore'],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = p.communicate()
if out.strip():
print("There is already docker node running. Stop it using:")
print("nearup stop")
exit(1)
def stop_native():
if os.path.exists(NODE_PID):
with open(NODE_PID) as f:
for line in f.readlines():
pid, proc_name, _ = map(str.strip, line.strip(' \n').split("|"))
pid = int(pid)
if proc_name in proc_name_from_pid(pid):
print(f"Stopping process {proc_name} with pid", pid)
kill(pid, SIGTERM)
# Ensure the pid is killed, not just SIGTERM signal sent
while True:
try:
os.kill(pid, 0)
except OSError:
break
unlink(NODE_PID)
def stop_watcher():
try:
with open(os.path.expanduser(f'~/.nearup/watcher.pid')) as f:
pid = int(f.read())
kill(pid, SIGTERM)
print(f'Stopping near watcher with pid {pid}')
os.remove(os.path.expanduser(f'~/.nearup/watcher.pid'))
except OSError:
pass
except FileNotFoundError:
pass
else:
while True:
try:
os.kill(pid, 0)
except OSError:
break
def run_docker_testnet(image,
home,
*,
shards=None,
validators=None,
non_validators=None):
subprocess.check_output(['mkdir', '-p', home])
home = os.path.abspath(home)
command = [
'docker', 'run', '-u', USER, '--rm', '-v',
'%s:/srv/near' % home, image, 'near', '--home', '/srv/near', 'testnet'
]
if shards:
command.extend(['--shards', str(shards)])
if validators:
command.extend(['--v', str(validators)])
if non_validators:
command.extend(['--n', str(non_validators)])
try:
subprocess.check_output(command)
except subprocess.CalledProcessError as e:
print('Failed to run docker near. Error:', file=sys.stderr)
print(e.stderr, file=sys.stderr)
exit(1)
except FileNotFoundError as exc:
print(
"Failed to run docker near: docker is not installed or not in PATH",
file=sys.stderr)
exit(1)
def run_nodocker(home_dir,
binary_path,
boot_nodes,
verbose,
chain_id,
watch=False):
"""Runs NEAR core outside of docker."""
print("Starting NEAR client...")
os.environ['RUST_BACKTRACE'] = '1'
pid_fd = open(NODE_PID, 'w')
# convert verbose = True to --verbose '' command line argument
if verbose:
verbose = ''
else:
verbose = None
LOGS_FOLDER = os.path.expanduser('~/.nearup/logs')
subprocess.check_output(['mkdir', '-p', LOGS_FOLDER])
proc = run_binary(os.path.join(binary_path, 'near'),
home_dir,
'run',
verbose=verbose,
boot_nodes=boot_nodes,
output=os.path.join(LOGS_FOLDER, chain_id),
watch=watch)
proc_name = proc_name_from_pid(proc.pid)
print(proc.pid, "|", proc_name, "|", chain_id, file=pid_fd)
pid_fd.close()
print(
"Node is running! \nTo check logs call: `nearup logs` or `nearup logs --follow`"
)
def __init__(self):
parser = argparse.ArgumentParser(description='Nearup',
usage='''nearup <command> [<args>]
Commands are:
mainnet (Coming soon) Run a mainnet node
testnet Run a testnet node
betanet Run a betanet node
localnet Run a network with several nodes locally
stop Stop the currently running node
logs Show logs of currently running non docker node
Run nearup <command> --help to see help for specific command
''')
parser.add_argument('command', help='Subcommand to run')
args = parser.parse_args(sys.argv[1:2])
if not hasattr(self, args.command):
print('Unrecognized command', file=sys.stderr)
parser.print_help(sys.stderr)
exit(2)
self.command = args.command
getattr(self, args.command)()
def generate_node_key(home, binary_path, nodocker, image):
print("Generating node key...")
if nodocker:
cmd = [f'{binary_path}/keypair-generator']
cmd.extend(['--home', home])
cmd.extend(['--generate-config'])
cmd.extend(['node-key'])
try:
subprocess.call(cmd)
except KeyboardInterrupt:
print("\nStopping NEARCore.")
else:
subprocess.check_output(['mkdir', '-p', home])
subprocess.check_output([
'docker', 'run', '-u', USER, '-v',
'%s:/srv/keypair-generator' % home, image, 'keypair-generator',
'--home=/srv/keypair-generator', '--generate-config', 'node-key'
])
print("Node key generated")
def run_docker(image,
home_dir,
boot_nodes,
verbose,
container_name='nearcore',
host_network=False,
watch=False):
"""Runs NEAR core inside the docker container"""
print("Starting NEAR client docker...")
docker_stop_if_exists(container_name)
# Start nearcore container, mapping home folder and ports.
envs = [
'-e', 'RUST_BACKTRACE=1', '-e',
'RUST_LOG=%s' % os.environ.get('RUST_LOG', '')
]
home_dir = os.path.abspath(home_dir)
rpc_port = get_port(home_dir, 'rpc')
network_port = get_port(home_dir, 'network')
cmd = ['near', '--home', '/srv/near']
if verbose:
cmd += ['--verbose', '']
cmd.append('run')
if boot_nodes:
cmd.append('--boot-nodes=%s' % boot_nodes)
if host_network:
network_flags = ['--network', 'host']
else:
network_flags = ['-p', rpc_port, '-p', network_port]
subprocess.check_output(['mkdir', '-p', home_dir])
try:
subprocess.check_output([
'docker', 'run', '-u', USER, '-d', '-v',
'%s:/srv/near' %
home_dir, *network_flags, '-v', '/tmp:/tmp', '--ulimit', 'core=-1',
'--name', container_name, '--restart', 'unless-stopped'
] + envs + [image] + cmd)
except subprocess.CalledProcessError as e:
print('Failed to run docker near. Error:', file=sys.stderr)
print(e.stderr, file=sys.stderr)
exit(1)
if watch:
run_watcher(watch, 'docker')
def generate_signer_key(home, binary_path, nodocker, image, account_id):
print("Generating signer keys...")
if nodocker:
cmd = [f'{binary_path}/keypair-generator']
cmd.extend(['--home', home])
cmd.extend(['--generate-config'])
cmd.extend(['--account-id', account_id])
cmd.extend(['signer-keys'])
try:
subprocess.call(cmd)
except KeyboardInterrupt:
print("\nStopping NEARCore.")
else:
subprocess.check_output(['mkdir', '-p', home])
subprocess.check_output([
'docker', 'run', '-u', USER, '-v',
'%s:/srv/keypair-generator' % home, image, 'keypair-generator',
'--home=/srv/keypair-generator', '--generate-config',
'--account-id=%s' % account_id, 'signer-keys'
])
print("Signer keys generated")
def nearup_restart(args):
main_script = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'main.py'))
subprocess.check_output(['python3', main_script, 'stop', '--keep-watcher'])
subprocess.Popen(['python3', main_script, *args])
print('done')
def nearup_restart(args):
main_script = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'main.py'))
subprocess.check_output(['python3', main_script, 'stop', '--keep-watcher'])
subprocess.Popen(['python3', main_script, *args])
print('done')
if __name__ == '__main__':
net = sys.argv[1]
home_dir = sys.argv[2]
docker = sys.argv[3]
args = sys.argv[4:]
latest_deploy_at = get_latest_deploy_at(net)
while True:
time.sleep(60)
try:
new_latest_deploy_at = get_latest_deploy_at(net)
if new_latest_deploy_at and new_latest_deploy_at != latest_deploy_at:
print(
f'new deploy happens at {new_latest_deploy_at}, restart nearup'
)
nearup_restart(args)
break
except:
traceback.print_exc()
pass
def check_and_setup(nodocker,
binary_path,
image,
home_dir,
init_flags,
no_gas_price=False):
"""Checks if there is already everything setup on this machine, otherwise sets up NEAR node."""
chain_id = get_chain_id_from_flags(init_flags)
if os.path.exists(os.path.join(home_dir)):
missing = []
for file in ['node_key.json', 'config.json', 'genesis.json']:
if not os.path.exists(os.path.join(home_dir, file)):
missing.append(file)
if missing:
print(
f'Missing files {", ".join(missing)} in {home_dir}. Maybe last init was failed. either specify different --home or remove {home_dir} to start from scratch.',
file=sys.stderr)
exit(1)
genesis_config = json.loads(
open(os.path.join(os.path.join(home_dir, 'genesis.json'))).read())
if genesis_config['chain_id'] != chain_id:
print(
f"Folder {home_dir} already has network configuration for {genesis_config['chain_id']}\n"
f"You want to run {chain_id}, either specify different --home or remove {home_dir} to start from scratch.",
file=sys.stderr)
exit(1)
if chain_id == 'devnet':
if check_and_update_genesis(chain_id, home_dir):
# For devnet, also update config because boot nodes changed
print(f'Update devnet config for new boot nodes')
os.remove(os.path.join(home_dir, 'config.json'))
download_config('devnet', home_dir)
elif chain_id in ['betanet', 'testnet']:
check_and_update_genesis(chain_id, home_dir)
else:
print(f'Start {chain_id}')
print("Using existing node configuration from %s for %s" %
(home_dir, genesis_config['chain_id']))
return
print("Setting up network configuration.")
account_id = [x for x in init_flags if x.startswith('--account-id')]
if not account_id:
prompt = "Enter your account ID"
if chain_id != '':
prompt += " (leave empty if not going to be a validator): "
else:
prompt += ": "
account_id = input(prompt)
init_flags.append('--account-id=%s' % account_id)
else:
account_id = account_id[0].split('=')[-1]
if nodocker:
if chain_id in ['devnet', 'betanet', 'testnet']:
nodocker_init_official(chain_id, binary_path, home_dir, account_id)
else:
nodocker_init(home_dir, binary_path, init_flags)
else:
if chain_id in ['devnet', 'betanet', 'testnet']:
docker_init_official(chain_id, image, home_dir, account_id)
else:
docker_init(image, home_dir, init_flags)
if chain_id not in ['devnet', 'betanet', 'testnet'] and no_gas_price:
filename = os.path.join(home_dir, 'genesis.json')
genesis_config = json.load(open(filename))
genesis_config['gas_price'] = 0
genesis_config['min_gas_price'] = 0
json.dump(genesis_config, open(filename, 'w'))
elif no_gas_price:
print(
f'no_gas_price is only for local development network, ignoring for {chain_id}'
)
help='Follow logs')
parser.add_argument(
'-n',
help='show given number of lines of logs (Default: 100)',
default=100,
type=int)
self.args = parser.parse_args(sys.argv[2:])
if __name__ == '__main__':
sys.argv[0] = 'nearup'
nearup_arg_parser = NearupArgParser()
command, args = nearup_arg_parser.command, nearup_arg_parser.args
if command in ['devnet', 'betanet', 'testnet']:
if args.local:
print("Flag --local deprecated, please use --nodocker")
nodocker = args.nodocker or args.local
args.home = os.path.abspath(args.home)
init_flags = [f'--chain-id={command}']
if args.account_id:
init_flags.append(f'--account-id={args.account_id}')
setup_and_run(nodocker,
args.binary_path,
args.image,
args.home,
init_flags=init_flags,
boot_nodes=args.boot_nodes,
verbose=args.verbose,
args=sys.argv[1:])
if command == 'mainnet':
print(
def stop_docker(containers):
"""Stops docker for Nearcore if they are running."""
print('Stopping docker near')
docker_stop_if_exists(containers)
def setup_and_run(nodocker,
binary_path,
image,
home_dir,
init_flags,
boot_nodes,
verbose=False,
no_gas_price=False,
args=None):
check_exist_neard()
chain_id = get_chain_id_from_flags(init_flags)
if nodocker:
if binary_path == '':
print(f'Using officially compiled binary')
uname = os.uname()[0]
if uname not in ['Linux', 'Darwin']:
print(
'Sorry your Operating System does not have officially compiled binary now.\nPlease compile locally by `make debug` or `make release` in nearcore and set --binary-path'
)
exit(1)
binary_path = os.path.expanduser(f'~/.nearup/near/{chain_id}')
subprocess.check_output(['mkdir', '-p', binary_path])
download_binary(chain_id, uname)
watch = {"args": args, "net": chain_id, 'home': home_dir}
else:
print(f'Using local binary at {binary_path}')
check_binary_version(binary_path, chain_id)
watch = False
else:
if image == 'auto':
image = net_to_docker_image(chain_id)
watch = {"args": args, "net": chain_id, 'home': home_dir}
else:
watch = False
try:
print(f'Pull docker image {image}')
subprocess.check_output(['docker', 'pull', image])
except subprocess.CalledProcessError as exc:
print("Failed to fetch docker containers: \n%s" % exc.stderr,
file=sys.stderr)
exit(1)
except FileNotFoundError as exc:
print(
"Failed to fetch docker containers, docker is not installed or not in PATH",
file=sys.stderr)
exit(1)
check_and_setup(nodocker, binary_path, image, home_dir, init_flags,
no_gas_price)
print_staking_key(home_dir)
if nodocker:
run_nodocker(home_dir,
binary_path,
boot_nodes,
verbose,
chain_id,
watch=watch)
else:
run_docker(image, home_dir, boot_nodes, verbose, watch=watch)
print(
"Node is running! \nTo check logs call: docker logs --follow nearcore"
)
