def jumpstart(self, name, network_directory, block):
self.app.log.info(f'Attempting to jumpstart {name} to block: {block}.')
url = f'{self.app.config["hydra"]["channel_url"]}/jumpstart/{name}/jumps.json'
# Get the published jumpstart data
try:
jumps_json = json.loads(requests.get(url).content)
except Exception as exc: # pylint: disable=broad-except
self.app.log.debug(f'Jumpstart metadata retrieval failed with: {exc}')
self.app.log.warning(f'No jumpstart data found for network {name}. Continuing without jumpstart')
return
if block not in jumps_json:
raise HydraError(f'Network {name} does not have jumpstart for block {block}')
# Next operations will all occur within the node directory for this network
os.chdir(network_directory)
# Get the jumpstart gzipped tar file
url = f'{self.app.config["hydra"]["channel_url"]}/jumpstart/{name}/{jumps_json[block]}'
jumpstart_tarfile = jumps_json[block]
try:
self.app.utils.download_file_stream(jumpstart_tarfile, url)
except Exception as exc:
raise HydraError(f'Unable to download jumpstart file {jumpstart_tarfile}: {exc}')
# Cleanup existing data that we're overwriting from jumpstart
for delete_dir in ['app.db', 'receipts_db', 'chaindata/data']:
try:
rmtree(os.path.join(network_directory, delete_dir))
except FileNotFoundError as exc:
self.app.log.debug(f'{exc}')
except IOError as exc:
raise HydraError(f'Cleanup of existing data failed: {exc}')
# Extract jumpstart over network directory
self.app.log.info(f'Extracting jumpstart contents')
try:
with tarfile.open(jumpstart_tarfile) as tar:
tar_members = tar.getmembers()
for member in tqdm(iterable=tar_members, total=(len(tar_members) - 1)):
tar.extract(member=member)
except tarfile.TarError as exc:
raise HydraError(f'Unable to extract jumpstart file {jumpstart_tarfile}: {exc}')
# Cleanup jumpstart gzipped tar file
try:
os.remove(jumpstart_tarfile)
except OSError as exc:
self.app.log.warning(f'Unable to cleanup jumpstart tar. {exc}')
self.app.log.info(f'Jumpstarting network complete!')
def env_or_arg(self, arg_name, env_name, or_path=None, required=False):
# pargs has default value of None if argument not provided
value = getattr(self.app.pargs, arg_name)
if value is None:
if env_name in os.environ:
value = os.environ[env_name]
self.app.log.info(
f'--{arg_name} not specified, using environ[{env_name}]: {value}'
)
elif or_path and os.path.exists(self.app.utils.path(or_path)):
with open(self.app.utils.path(or_path), 'r') as variable_file:
value = variable_file.read().strip()
self.app.log.info(
f'--{arg_name} not specified, using file {or_path}: {value}'
)
if required and value is None:
self.app.log.error(
f'You must specify either --{arg_name} or set {env_name} in your environment'
)
raise HydraError(
f'You must specify either --{arg_name} or set {env_name} in your environment'
)
return value
def pull_registry(self):
name = self.app.utils.env_or_arg('name',
'HYDRA_NETWORK',
or_path='.hydra_network')
try:
url = f'{self.app.config["hydra"]["channel_url"]}/networks/{name}/hydra.json'
network_registry = json.loads(requests.get(url).content)
self.app.network.register(name, network_registry)
except Exception as exc:
raise HydraError(
f'Unable to pull updated registry information: {exc}')
def uninstall_systemd(self, name, binary='shipchain'):
service_name = f'{name}{"" if binary is "shipchain" else f".{binary}"}.service'
systemd_service = f'/etc/systemd/system/{service_name}'
self.app.log.info(f'Uninstalling {service_name}')
if not os.path.exists(systemd_service):
raise HydraError(f'Systemd file {systemd_service} not found')
self.app.utils.binary_exec('sudo', 'systemctl', 'stop', service_name)
self.app.utils.binary_exec('sudo', 'systemctl', 'disable', service_name)
self.app.utils.binary_exec('sudo', 'systemctl', 'reset-failed', service_name, ignore_error=True)
self.app.utils.binary_exec('sudo', 'systemctl', 'daemon-reload')
self.app.utils.binary_exec('sudo', 'rm', systemd_service)
def leave_network(self):
name = self.app.utils.env_or_arg('name', 'HYDRA_NETWORK', or_path='.hydra_network', required=True)
destination = self.app.pargs.destination or self.app.utils.path(name)
# Verify network directory exists before we start removing everything
if not os.path.exists(destination):
raise HydraError(f'Network directory {destination} does not exist')
# Stop and remove service before removing network directory
try:
self.app.client.uninstall_systemd(name)
except HydraError as exc:
self.app.log.warning(exc)
self.app.log.info(f'Service not installed. Attempting to stop executable manually.')
self.app.client.find_and_kill_executable(destination)
# Remove network directory
rmtree(destination)
self.app.log.info(f'Removed network directory {destination}')
# Remove .hydra_network if this is the default network
default_network_file = self.app.utils.path('.hydra_network')
if os.path.exists(default_network_file):
remove_default_network = False
with open(default_network_file, 'r') as network_file:
default_network = network_file.readline().strip()
if default_network == name:
remove_default_network = True
if remove_default_network:
os.remove(default_network_file)
self.app.log.info(f'Removed default network setting')
self.app.log.info(f'Successfully left network {name}')
def get(stub):
url = f'http://{host}:{port}{stub}'
try:
return json.loads(requests.get(url).content)
except requests.exceptions.ConnectionError:
raise HydraError(f'Error accessing {url}. Is your node running?')
def generate_jumpstart(self):
networks = self.app.network.read_networks_file()
name = self.app.utils.env_or_arg('name',
'HYDRA_NETWORK',
or_path='.hydra_network')
if name not in networks:
self.app.log.error(
f'You must choose a valid network name: {networks.keys()}')
return
if len(networks[name]['ips']) <= 1:
self.app.log.error(
f'Jumpstart loom.yaml would contain Oracle specific settings.')
raise HydraError(f'Not enough nodes in network')
ip = networks[name]['ips'][-1]
# We want to include current block height in tarfile name
self.app.log.info(f'Getting client status on {ip}')
block_height = json.loads(
self.app.network.run_command(
ip, 'hydra -o json client status'))["node_block_height"]
self.app.log.info(f'Current block height {block_height}')
# We don't want to package live databases
self.app.log.info(f'Stopping node before packaging jumpstart')
self.app.network.run_command(
ip, f'hydra client stop-service --name {name} 2>&1')
try:
tarfile = f'{datetime.today().strftime("%Y-%m-%d")}_{block_height}_{name}.tar.gz'
s3_destination = f's3://{self.app.release.dist_bucket}/jumpstart/{name}/{tarfile}'
jumpstart_include = [
'genesis.json',
'app.db',
'evm.db',
'receipts_db',
'chaindata/config/genesis.json',
'chaindata/data/blockstore.db',
'chaindata/data/evidence.db',
'chaindata/data/fnConsensus.db',
'chaindata/data/state.db',
'chaindata/data/tx_index.db',
]
self.app.log.info(f'Building {tarfile}')
self.app.network.run_command(
ip, f"cd /data/{name}; "
f"tar -zcf {tarfile} "
f"{' '.join(jumpstart_include)}")
# The AMI does not include awscli by default
self.app.log.info(f'Ensuring AWS CLI is available')
self.app.network.run_command(
ip, f"sudo apt-get -y install awscli 2>&1")
self.app.log.info(f'Uploading {tarfile} to {s3_destination}')
self.app.network.run_command(
ip,
f"cd /data/{name}; aws s3 cp {tarfile} {s3_destination} --acl public-read"
)
self.app.log.info(f'Cleaning up {tarfile}')
self.app.network.run_command(ip,
f"cd /data/{name}; rm -f {tarfile}")
s3 = self.app.release.get_boto().resource('s3')
try:
jumps_obj = s3.Object(self.app.release.dist_bucket,
f'jumpstart/{name}/jumps.json').get()
jumps_json = json.loads(
jumps_obj['Body'].read().decode('utf-8'))
except s3.meta.client.exceptions.NoSuchKey:
jumps_json = {}
jumps_json[block_height] = tarfile
jumps_json['latest'] = tarfile
s3.Object(self.app.release.dist_bucket,
f'jumpstart/{name}/jumps.json').put(
ACL='public-read',
Body=json.dumps(jumps_json).encode('utf-8'),
ContentType='application/json',
)
self.app.log.info(f'Jumpstart generation complete!')
except Exception as exc:
self.app.log.error(f'Jumpstart generation failed {exc}')
finally:
self.app.log.info(f'Restarting node service')
self.app.network.run_command(
ip, f'hydra client start-service --name {name} 2>&1')
def monitor_cloudformation_stack(self, stack, node_count, name):
stack.reload()
registry = {
'bootstrapped': datetime.utcnow().strftime('%c'),
'size': node_count,
'status': stack.stack_status,
'outputs': {},
'ips': [],
'node_data': {}
}
self.app.network.register(name, registry)
while stack.stack_status == 'CREATE_IN_PROGRESS':
self.app.log.info(f'Status: {stack.stack_status}')
time.sleep(10)
stack.reload()
registry['status'] = stack.stack_status
self.app.network.register(name, registry)
self.app.log.info(f'Status: {stack.stack_status}')
if stack.stack_status != 'CREATE_COMPLETE':
user_response = shell.Prompt(
'Error deploying cloudformation, what do you want to do?',
options=['Delete It', 'Leave It'],
numbered=True)
if user_response.prompt() == 'Delete It':
stack.delete()
return
if self.app.pargs.default:
with open(self.app.utils.path('.hydra_network'),
'w+') as network_file:
network_file.write(name)
registry['outputs'] = {
o['OutputKey']: o['OutputValue']
for o in stack.outputs
}
for node in range(node_count):
ip = registry['outputs'][f'IP{node}']
registry['ips'].append(ip)
self.app.log.info(f'Node IP: {ip}')
self.app.network.register(name, registry)
self.app.log.info(
'Creation complete, pausing for a minute while the software installs...'
)
bootstrapped_a_node = False
for ip in registry['ips']:
for attempt in range(1, 11):
try:
self.app.log.info(
f'Bootstrapping node {ip} attempt {attempt}...')
registry['node_data'][ip] = self.get_bootstrap_data(
ip, name)
bootstrapped_a_node = True
break
except Exception: # pylint: disable=broad-except
if attempt >= 10:
self.app.log.error(
f'Timed out waiting for node {ip} to bootstrap.')
break
time.sleep(30)
continue
if not bootstrapped_a_node:
raise HydraError(f'Bootstrapping failed for all nodes')
registry['bootstrapped'] = datetime.utcnow().strftime('%c')
self.app.network.register(name, registry)
self.app.log.info('Stack launch success!')