def get_field(self, field):
# Ensure field name is valid
if field not in self.FIELDS:
return None
with no_print():
# Find, parse and validate configs
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
if field == 'name':
return portal_name
if field == 'user':
return portal_spec['spot_instance']['remote_user']
if field == 'key':
return portal_spec['spot_instance']['identity_file']
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'],
config['aws_region'])
# Get current user
aws_user = aws.get_user_identity()
# Get spot instance
instance_info = aws.find_spot_instance(portal_name,
aws_user['Arn'])
if field == 'status':
return 'open' if instance_info is not None else 'close'
# If portal is closed, we cannot provide any other information
if instance_info is None:
return None
if field == 'id':
return instance_info['InstanceId']
if field == 'type':
return instance_info['InstanceType']
if field == 'host':
return instance_info['PublicDnsName']
if field == 'ip':
return instance_info['PublicIpAddress']
if field == 'remote':
return '{}@{}'.format(
portal_spec['spot_instance']['remote_user'],
instance_info['PublicDnsName'])
return None
def run(self):
# Find, parse and validate configs
with no_print():
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'],
config['aws_region'])
# Get current user
aws_user = aws.get_user_identity()
# Get spot instance
instance_info = aws.find_spot_instance(portal_name, aws_user['Arn'])
if instance_info is None:
raise CommandError(
'Portal `{}` does not seem to be opened'.format(portal_name))
# Get values for ssh
key_file = portal_spec['spot_instance']['identity_file']
user = portal_spec['spot_instance']['remote_user']
host = instance_info['PublicDnsName']
print('Connecting to the remote machine...')
print('\tssh -i "{}" {}@{}'.format(key_file, user, host).expandtabs(4))
# If requested, configure a preamble (a set of commands to be run automatically after connection)
preamble = []
if self._args.tmux is not None:
preamble = [
'-t',
'""tmux attach-session -t {sess} || tmux new-session -s {sess}""'
.format(sess=self._args.tmux)
]
print('Upon connection will open tmux session `{}`.'.format(
self._args.tmux))
print('')
# Ssh to remote host (effectively replace current process by ssh)
os.execvp('ssh', ['ssh', '-i', key_file, '{}@{}'.format(user, host)] +
preamble)
def run(self):
# Find, parse and validate configs
with print_scope('Checking configuration:', 'Done.\n'):
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'], config['aws_region'])
with print_scope('Retrieving data from AWS:', 'Done.\n'):
# Get current user
with step('Get user identity'):
user = aws.get_user_identity()
# Get spot instance
with step('Get spot instance', error_message='Portal `{}` does not seem to be opened'.format(portal_name),
catch=[RuntimeError]):
spot_instance = common.get_spot_instance(aws, portal_name, user['Arn'])
spot_fleet_request_id = \
filter(lambda tag: tag['Key'] == 'aws:ec2spot:fleet-request-id', spot_instance['Tags'])[0]['Value']
# Get spot instance
with step('Get spot request', error_message='Portal `{}` does not seem to be opened'.format(portal_name),
catch=[RuntimeError]):
spot_fleet_request = common.get_spot_fleet_request(aws, spot_fleet_request_id)
# TODO: print fleet and instance statistics
# Cancel spot instance request
aws.cancel_spot_fleet_request(spot_fleet_request_id)
# Clean up volumes' tags
volume_ids = [volume['Ebs']['VolumeId']
for volume in spot_instance['BlockDeviceMappings']
if not volume['Ebs']['DeleteOnTermination']]
aws.remove_tags(volume_ids, 'mount-point')
print('Portal `{}` has been closed.'.format(portal_name))
def run(self):
# Find, parse and validate configs
with print_scope('Checking configuration:', 'Done.\n'):
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
# Ensure there is at least one channel spec
with step('Check specifications for channels',
error_message=
'Portal specification does not contain any channel'):
channels = portal_spec['channels']
if len(channels) == 0:
raise Exception()
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'],
config['aws_region'])
with print_scope('Retrieving data from AWS:', 'Done.\n'):
# Get current user
with step('Get user identity'):
user = aws.get_user_identity()
# Get spot instance
with step('Get spot instance',
error_message='Portal `{}` does not seem to be opened'.
format(portal_name),
catch=[RuntimeError]):
spot_instance = common.get_spot_instance(
aws, portal_name, user['Arn'])
host_name = spot_instance['PublicDnsName']
# Print information about the channels
with print_scope(
'Channels defined for portal `{}`:'.format(portal_name), ''):
for i in range(len(channels)):
channel = channels[i]
with print_scope('Channel #{} ({}):'.format(
i, channel['direction'].upper())):
print('Local: {}'.format(channel['local_path']))
print('Remote: {}'.format(channel['remote_path']))
# Specify remote host for ssh
env.user = portal_spec['spot_instance']['remote_user']
env.key_filename = [portal_spec['spot_instance']['identity_file']]
env.hosts = [host_name]
# Periodically sync files across all channels
print('Syncing... (press ctrl+C to interrupt)')
for channel in channels:
is_upload = channel['direction'] == 'out'
is_recursive = channel[
'recursive'] if 'recursive' in channel else False
delay = 1.0
if 'delay' in channel:
delay = channel['delay']
run_periodically(sync_files, [
channel['local_path'], channel['remote_path'], is_upload,
is_recursive
], delay)
def run(self):
# Find, parse and validate configs
with print_scope('Checking configuration:', 'Done.\n'):
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
instance_spec = portal_spec['spot_instance']
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'],
config['aws_region'])
with print_scope('Retrieving data from AWS:', 'Done.\n'):
# Get current user
with step('Get user identity'):
user = aws.get_user_identity()
# Ensure that instance does not yet exist
with step('Check already running instances',
error_message='Portal `{}` seems to be already opened'.
format(portal_name),
catch=[RuntimeError]):
common.check_instance_not_exists(aws, portal_name, user['Arn'])
# Ensure persistent volumes are available
with step('Check volumes availability', catch=[RuntimeError]):
volume_ids = [
volume_spec['volume_id']
for volume_spec in portal_spec['persistent_volumes']
]
common.check_volumes_availability(aws, volume_ids)
# If subnet Id is not provided, pick the default subnet of the availability zone
if 'subnet_id' not in instance_spec or not instance_spec[
'subnet_id']:
with step('Get subnet id', catch=[IndexError, KeyError]):
subnets = aws.get_subnets(
instance_spec['availability_zone'])
instance_spec['subnet_id'] = subnets[0]['SubnetId']
# Make request for Spot instance
instance_type = instance_spec['instance_type']
with print_scope('Requesting a Spot instance of type {}:'.format(
instance_type)):
request_config = aws_helpers.single_instance_spot_fleet_request(
portal_spec, portal_name, user['Arn'])
response = aws.request_spot_fleet(request_config)
spot_fleet_request_id = response['SpotFleetRequestId']
# Wait for spot fleet request to be fulfilled
print('Waiting for the Spot instance to be created...')
print(
'(usually it takes around a minute, but might take much longer)'
)
begin_time = datetime.datetime.now()
next_time = begin_time
try:
while True:
# Repeat status request every N seconds
if datetime.datetime.now() > next_time:
spot_fleet_request = aws.get_spot_fleet_request(
spot_fleet_request_id)
next_time += datetime.timedelta(seconds=5)
# Compute time spend in waiting
elapsed = datetime.datetime.now() - begin_time
# Check request state and activity status
request_state = spot_fleet_request['SpotFleetRequestState']
if request_state == 'active':
spot_request_status = spot_fleet_request[
'ActivityStatus']
if spot_request_status == 'fulfilled':
break
else:
print(
'Elapsed {}s. Spot request is {} and has status `{}`'
.format(elapsed.seconds, request_state,
spot_request_status),
end='\r')
else:
print('Elapsed {}s. Spot request is {}'.format(
elapsed.seconds, request_state),
end='\r')
sys.stdout.flush() # ensure stdout is flushed immediately.
time.sleep(0.5)
except KeyboardInterrupt:
print('\n')
print('Interrupting...')
# Cancel spot instance request
aws.cancel_spot_fleet_request(spot_fleet_request_id)
raise CommandError('Spot request has been cancelled.')
print('\nSpot instance is created in {} seconds.\n'.format(
(datetime.datetime.now() - begin_time).seconds))
# Get id of the created instance
spot_fleet_instances = aws.get_spot_fleet_instances(
spot_fleet_request_id)
instance_id = spot_fleet_instances[0]['InstanceId']
# Get information about the created instance
instance_info = aws.get_instance(instance_id)
# Make requests to attach persistent volumes
with print_scope('Attaching persistent volumes:'):
for volume_spec in portal_spec['persistent_volumes']:
response = aws.attach_volume(instance_id,
volume_spec['volume_id'],
volume_spec['device'])
# Check status code
if response['State'] not in ['attaching', 'attached']:
raise CommandError(
'Could not attach persistent volume `{}`'.format(
volume_spec['volume_id']))
# Wait for persistent volumes to be attached
print('Waiting for the persistent volumes to be attached...')
begin_time = datetime.datetime.now()
next_time = begin_time
while True:
# Repeat status request every N seconds
if datetime.datetime.now() > next_time:
volumes = aws.get_volumes_by_id(volume_ids)
next_time += datetime.timedelta(seconds=1)
# Compute time spend in waiting
elapsed = datetime.datetime.now() - begin_time
if all([
volume['Attachments'][0]['State'] == 'attached'
for volume in volumes
]):
break
else:
states = [
'{} - `{}`'.format(volume['VolumeId'],
volume['Attachments'][0]['State'])
for volume in volumes
]
print('Elapsed {}s. States: {}'.format(
elapsed.seconds, ', '.join(states)),
end='\r')
sys.stdout.flush() # ensure stdout is flushed immediately.
time.sleep(0.5)
print('\nPersistent volumes are attached in {} seconds.\n'.format(
(datetime.datetime.now() - begin_time).seconds))
# Configure ssh connection via fabric
env.user = instance_spec['remote_user']
env.key_filename = [instance_spec['identity_file']]
env.hosts = instance_info['PublicDnsName']
env.connection_attempts = self._fabric_retry_limit
with print_scope('Preparing the instance:', 'Instance is ready.\n'):
# Mount persistent volumes
for i in range(len(portal_spec['persistent_volumes'])):
with step('Mount volume #{}'.format(i),
error_message='Could not mount volume',
catch=[RuntimeError]):
volume_spec = portal_spec['persistent_volumes'][i]
# Mount volume
with hide('running', 'stdout'):
execute(self.mount_volume, volume_spec['device'],
volume_spec['mount_point'],
instance_spec['remote_group'],
instance_spec['remote_user'])
# Store extra information in volume's tags
aws.add_tags(volume_spec['volume_id'],
{'mount-point': volume_spec['mount_point']})
# TODO: consider importing and executing custom fab tasks instead
# Install extra python packages, if needed
if 'extra_python_packages' in instance_spec and len(
instance_spec['extra_python_packages']) > 0:
with step('Install extra python packages',
error_message='Could not install python packages',
catch=[RuntimeError]):
python_packages = instance_spec['extra_python_packages']
virtual_env = instance_spec['python_virtual_env']
with hide('running', 'stdout'):
execute(self.install_python_packages, python_packages,
virtual_env)
# Print summary
print('Portal `{}` is now opened.'.format(portal_name))
with print_scope('Summary:', ''):
with print_scope('Instance:'):
print('Id: {}'.format(instance_id))
print('Type: {}'.format(
instance_info['InstanceType']))
print('Public IP: {}'.format(
instance_info['PublicIpAddress']))
print('Public DNS name: {}'.format(
instance_info['PublicDnsName']))
with print_scope('Persistent volumes:'):
for volume_spec in portal_spec['persistent_volumes']:
print('{}: {}'.format(volume_spec['device'],
volume_spec['mount_point']))
# Print ssh command
print('Use the following command to connect to the remote machine:')
print('ssh -i "{}" {}@{}'.format(instance_spec['identity_file'],
instance_spec['remote_user'],
instance_info['PublicDnsName']))
def show_full_info(self):
# Find, parse and validate configs
with print_scope('Checking configuration:', 'Done.\n'):
config = get_config(self._args)
portal_spec, portal_name = get_portal_spec(self._args)
# Create AWS client
aws = AwsClient(config['aws_access_key'], config['aws_secret_key'],
config['aws_region'])
volumes = []
with print_scope('Retrieving data from AWS:', 'Done.\n'):
# Get current user
with step('Get user identity'):
aws_user = aws.get_user_identity()
# Get spot instance
with step('Get spot instance',
error_message='Portal `{}` does not seem to be opened'.
format(portal_name),
catch=[RuntimeError]):
instance_info = aws.find_spot_instance(portal_name,
aws_user['Arn'])
# Get persistent volumes, if portal is opened
if instance_info is not None:
with step('Get volumes'):
volume_ids = [
volume['Ebs']['VolumeId']
for volume in instance_info['BlockDeviceMappings']
if not volume['Ebs']['DeleteOnTermination']
]
volumes = aws.get_volumes_by_id(volume_ids)
# Print status
if instance_info is not None:
with print_scope('Summary:', ''):
print('Name: {}'.format(portal_name))
print('Status: open')
with print_scope('Instance:', ''):
print('Id: {}'.format(
instance_info['InstanceId']))
print('Type: {}'.format(
instance_info['InstanceType']))
print('Public IP: {}'.format(
instance_info['PublicIpAddress']))
print('Public DNS name: {}'.format(
instance_info['PublicDnsName']))
print('User: {}'.format(
portal_spec['spot_instance']['remote_user']))
with print_scope('Persistent volumes:', ''):
for i in range(len(volumes)):
volume = volumes[i]
with print_scope('Volume #{}:'.format(i), ''):
self.print_volume_info(volume)
# Print ssh command
with print_scope(
'Use the following command to connect to the remote machine:'
):
print('ssh -i "{}" {}@{}'.format(
portal_spec['spot_instance']['identity_file'],
portal_spec['spot_instance']['remote_user'],
instance_info['PublicDnsName']))
else:
with print_scope('Summary:'):
print('Name: {}'.format(portal_name))
print('Status: close')