def collect_gcloud_config(num_nodes):
tempdir = get_near_tempdir()
keys = []
for i in range(num_nodes):
node_dir = tempdir / f'node{i}'
if not node_dir.exists():
# TODO: avoid hardcoding the username
logger.info(f'downloading node{i} config from gcloud')
node_dir.mkdir(parents=True, exist_ok=True)
host = gcloud.get(f'pytest-node-{user_name()}-{i}')
for filename in ('config.json', 'signer0_key.json',
'validator_key.json', 'node_key.json'):
host.download(f'/home/bowen_nearprotocol_com/.near/{filename}',
str(node_dir))
with open(node_dir / 'signer0_key.json') as f:
key = json.load(f)
keys.append(key)
with open(tempdir / 'node0' / 'config.json') as f:
config = json.load(f)
ip_addresses = map(lambda x: x.split('@')[-1],
config['network']['boot_nodes'].split(','))
res = {
'nodes':
list(map(lambda x: {
'ip': x.split(':')[0],
'port': 3030
}, ip_addresses)),
'accounts':
keys
}
outfile = tempdir / 'gcloud_config.json'
with open(outfile, 'w') as f:
json.dump(res, f)
os.environ[cluster.CONFIG_ENV_VAR] = str(outfile)
def test_create_delete_5_instance():
futures = []
machines = []
with Timer('create 5 instances'):
for i in range(5):
futures.append(
go(create_or_get,
name="test-rc-node-" + str(i),
machine_type="n1-standard-1",
disk_size="20G",
image_project='ubuntu-os-cloud',
image_family='ubuntu-1804-lts',
zone='us-west2-a',
preemptible=False,
firewall_allows=['tcp:8080']))
for f in as_completed(futures):
m = f.result()
print("Created machine:", m.name)
machines.append(m)
futures = {}
def delete_print(m):
m.delete()
print("Deleted machine:", m.name)
with Timer('delete 5 instances'):
pmap(delete_print, machines)
for m in machines:
assert gcloud.get(m.name) is None
def test_tmux():
machine = gcloud.create(name="test-rc-node", machine_type="n1-standard-1", disk_size="20G", image_project='ubuntu-os-cloud', image_family='ubuntu-1804-lts',
zone='us-west2-a', preemptible=False, firewall_allows=['tcp:8080'])
machine = gcloud.get('test-rc-node')
machine.run('sudo apt install tmux')
machine.run_detach_tmux('while true; do echo aaaaaa; sleep 10; done')
machine.kill_detach_tmux()
assert machine.run('cat /tmp/python-rc.log').stdout == 'aaaaaa\n'
machine.delete()
def collect_gcloud_config(num_nodes):
import pathlib
keys = []
for i in range(num_nodes):
if not os.path.exists(f'/tmp/near/node{i}'):
# TODO: avoid hardcoding the username
print(f'downloading node{i} config from gcloud')
pathlib.Path(f'/tmp/near/node{i}').mkdir(parents=True,
exist_ok=True)
gcloud.get(f'pytest-node-{user_name()}-{i}').download(
'/home/bowen_nearprotocol_com/.near/config.json',
f'/tmp/near/node{i}/')
gcloud.get(f'pytest-node-{user_name()}-{i}').download(
'/home/bowen_nearprotocol_com/.near/signer0_key.json',
f'/tmp/near/node{i}/')
gcloud.get(f'pytest-node-{user_name()}-{i}').download(
'/home/bowen_nearprotocol_com/.near/validator_key.json',
f'/tmp/near/node{i}/')
gcloud.get(f'pytest-node-{user_name()}-{i}').download(
'/home/bowen_nearprotocol_com/.near/node_key.json',
f'/tmp/near/node{i}/')
with open(f'/tmp/near/node{i}/signer0_key.json') as f:
key = json.load(f)
keys.append(key)
with open('/tmp/near/node0/config.json') as f:
config = json.load(f)
ip_addresses = map(lambda x: x.split('@')[-1],
config['network']['boot_nodes'].split(','))
res = {
'nodes':
list(
map(lambda x: {
'ip': x.split(':')[0],
'port': 3030
}, ip_addresses)),
'accounts':
keys
}
outfile = '/tmp/near/gcloud_config.json'
with open(outfile, 'w+') as f:
json.dump(res, f)
os.environ[CONFIG_ENV_VAR] = outfile
def __init__(self, *args, username=None, project=None, ssh_key_path=None):
if len(args) == 1:
name = args[0]
# Get existing instance assume it's ready to run.
self.instance_name = name
self.port = 24567
self.rpc_port = 3030
self.machine = gcloud.get(name,
username=username,
project=project,
ssh_key_path=ssh_key_path)
self.ip = self.machine.ip
elif len(args) == 4:
# Create new instance from scratch
instance_name, zone, node_dir, binary = args
self.instance_name = instance_name
self.port = 24567
self.rpc_port = 3030
self.node_dir = node_dir
self.machine = gcloud.create(
name=instance_name,
machine_type='n1-standard-2',
disk_size='50G',
image_project='gce-uefi-images',
image_family='ubuntu-1804-lts',
zone=zone,
firewall_allows=['tcp:3030', 'tcp:24567'],
min_cpu_platform='Intel Skylake',
preemptible=False,
)
# self.ip = self.machine.ip
self._upload_config_files(node_dir)
self._download_binary(binary)
with remote_nodes_lock:
global cleanup_remote_nodes_atexit_registered
if not cleanup_remote_nodes_atexit_registered:
atexit.register(atexit_cleanup_remote)
cleanup_remote_nodes_atexit_registered = True
else:
raise Exception()
def delete_machine(machine_name):
m = gcloud.get(machine_name)
if m:
m.delete()
def test_gcloud():
# Cleanup from last fail
a = gcloud.get('test-rc-node')
if a is not None:
a.delete()
# Test list
with Timer('list machine'):
old_machines = gcloud.list()
assert len(list(filter(lambda machine: machine.name ==
'test-rc-node', old_machines))) == 0
# Test get None
assert gcloud.get('test-rc-node') is None
# Test create machine
with Timer('create machine'):
machine1 = create_test_machine()
assert machine1.name == "test-rc-node"
assert machine1.ip != ''
assert machine1.ssh_key_path != ''
assert machine1.username != ''
assert machine1.status() == "RUNNING"
# Test get machine
with Timer('get machine'):
machine1_get = gcloud.get('test-rc-node')
assert machine1 == machine1_get
# Test list after create
new_machines = gcloud.list()
assert len(list(filter(lambda machine: machine.name ==
'test-rc-node', new_machines))) == 1
assert len(new_machines) == len(old_machines) + 1
# Test shutdown
with Timer('shutdown'):
machine1.shutdown()
assert machine1.status() == "TERMINATED"
# Test bootup
with Timer('bootup'):
machine1.bootup()
assert machine1.status() == "RUNNING"
# Test upload download
machine1.upload(local_path='rc/test/test_gcloud.py', machine_path="/tmp/")
assert machine1.run("ls /tmp/test_gcloud.py")
machine1.download(machine_path="/tmp/test_gcloud.py",
local_path="/tmp/test_gcloud.py")
assert open(
"/tmp/test_gcloud.py").read() == open("/tmp/test_gcloud.py").read()
run(['rm', '/tmp/test_gcloud.py'])
# Test run command on machine
machine1.run(['echo', 'aaa', '>', '/tmp/aaaa.txt'])
assert run('ls /tmp/aaaa.txt').returncode != 0
machine1.run(['grep', '', '/tmp/aaaa.txt'])
p = machine1.run('cat /tmp/aaaa.txt')
assert p.stdout == 'aaa\n'
# Test delete
with Timer('delete machine'):
machine1.delete()
assert gcloud.get('test-rc-node') is None
import sys
import json
import datetime
machines = gcloud.list()
pytest_nodes = list(
filter(lambda m: m.name.startswith('pytest-node'), machines))
test_id = datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d%H%M%S")
data = []
for node in pytest_nodes:
data.append({
'targets': [f'{node.ip}:3030'],
'labels': {
'test_id': test_id,
'name': node.name
}
})
with open('/tmp/near/targets.json', 'w') as f:
json.dump(data, f)
prometheus = gcloud.get('prometheus-grafana')
prometheus.upload('/tmp/near/targets.json', '/mnt/disks/sdb/prometheus')
prometheus.run('bash',
input='''
cd /mnt/disks/sdb/prometheus
docker stop prometheus
./start.sh
''')
def create_or_get(*, name, **kwargs):
return gcloud.get(name) or gcloud.create(name=name, **kwargs)
