def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
project_name = project_config['name']
region = instance_config['region']
local_ssh_port = instance_config['localSshPort']
# get instance IP address
stack = StackResource(None, project_name, region)
ec2 = boto3.client('ec2', region_name=region)
ip_address = get_instance_ip_address(ec2, stack.name)
if self._args.host_os:
# connect to the host OS
session_name = self._args.session_name if self._args.session_name else 'spotty-ssh-host-os'
remote_cmd = ['tmux', 'new', '-s', session_name, '-A']
else:
# connect to the container
session_name = self._args.session_name if self._args.session_name else 'spotty-ssh-container'
remote_cmd = ['tmux', 'new', '-s', session_name, '-A', 'sudo', '/scripts/container_bash.sh']
remote_cmd = subprocess.list2cmdline(remote_cmd)
# connect to the instance
ssh_command = get_ssh_command(project_name, region, ip_address, remote_cmd, local_ssh_port)
subprocess.call(ssh_command)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
region = instance_config['region']
project_name = project_config['name']
local_ssh_port = instance_config['localSshPort']
output.write('Syncing the project with S3 bucket...')
# sync the project with S3 bucket
sync_filters = project_config['syncFilters']
sync_project_with_s3(self._project_dir, project_name, region, sync_filters, output)
output.write('Syncing S3 bucket with the instance...')
# get instance IP address
stack = StackResource(None, project_name, region)
ec2 = boto3.client('ec2', region_name=region)
ip_address = get_instance_ip_address(ec2, stack.name)
# sync S3 with the instance
sync_instance_with_s3(ip_address, project_name, region, local_ssh_port)
output.write('Done')
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
project_name = project_config['name']
region = instance_config['region']
# get instance IP address
stack = StackResource(None, project_name, region)
ec2 = boto3.client('ec2', region_name=region)
ip_address = get_instance_ip_address(ec2, stack.name)
# connect to the instance
host = 'ubuntu@%s' % ip_address
key_path = KeyPairResource(None, project_name, region).key_path
ssh_command = [
'ssh', '-i', key_path, '-o', 'StrictHostKeyChecking no', '-t', host
]
if self._args.host_os:
session_name = self._args.session_name if self._args.session_name else 'spotty-ssh-host-os'
ssh_command += ['tmux', 'new', '-s', session_name, '-A']
else:
session_name = self._args.session_name if self._args.session_name else 'spotty-ssh-container'
ssh_command += [
'tmux', 'new', '-s', session_name, '-A', 'sudo',
'/scripts/container_bash.sh'
]
subprocess.call(ssh_command)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
project_name = project_config['name']
region = instance_config['region']
script_name = self._args.script_name
if script_name not in self._config['scripts']:
raise ValueError('Script "%s" is not defined in the configuration file.' % script_name)
# get instance IP address
stack = StackResource(None, project_name, region)
ec2 = boto3.client('ec2', region_name=region)
ip_address = get_instance_ip_address(ec2, stack.name)
# tmux session name
session_name = self._args.session_name if self._args.session_name else 'spotty-script-%s' % script_name
# base64 encoded user script from the configuration file
script_base64 = base64.b64encode(self._config['scripts'][script_name].encode('utf-8')).decode('utf-8')
# remote path where the script will be uploaded
script_path = '/tmp/docker/%s.sh' % script_name
# log file for the script outputs
script_log_file = '/var/log/spotty-run/%s.log' % script_name
# command to attach user to existing tmux session
attach_session_cmd = subprocess.list2cmdline(['tmux', 'attach', '-t', session_name, '>', '/dev/null', '2>&1'])
# command to upload user script to the instance
upload_script_cmd = subprocess.list2cmdline(['echo', script_base64, '|', 'base64', '-d', '>', script_path])
# command to log the time when user script started
start_time_cmd = subprocess.list2cmdline(['echo', '-e', '\\nScript started: `date \'+%Y-%m-%d %H:%M:%S\'`\\n',
'>>', script_log_file])
# command to run user script inside the docker container
docker_cmd = subprocess.list2cmdline(['sudo', '/scripts/container_bash.sh', '-xe', script_path, '2>&1',
'|', 'tee', '-a', script_log_file])
# command to create new tmux session and run user script
new_session_cmd = subprocess.list2cmdline(['tmux', 'new', '-s', session_name,
'%s && %s' % (start_time_cmd, docker_cmd)])
# composition of the commands: if user cannot be attached to the tmux session (assume the session doesn't
# exist), then we uploading user script to the instance, creating new tmux session and running that script
# inside the Docker container
remote_cmd = '%s || (%s && %s)' % (attach_session_cmd, upload_script_cmd, new_session_cmd)
# connect to the instance and run the command above
host = 'ubuntu@%s' % ip_address
key_path = KeyPairResource(None, project_name, region).key_path
ssh_command = ['ssh', '-i', key_path, '-o', 'StrictHostKeyChecking no', host, '-t', remote_cmd]
subprocess.call(ssh_command)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
region = instance_config['region']
project_name = project_config['name']
# create bucket for the project
s3 = boto3.client('s3', region_name=region)
project_bucket = BucketResource(s3, project_name, region)
bucket_name = project_bucket.create_bucket(output)
output.write('Syncing the project with S3 bucket...')
# sync the project with S3
project_filters = project_config['syncFilters']
AwsCli(region=region).s3_sync(self._project_dir,
's3://%s/project' % bucket_name,
delete=True,
filters=project_filters,
capture_output=False)
# get instance IP address
stack = StackResource(None, project_name, region)
ec2 = boto3.client('ec2', region_name=region)
ip_address = get_instance_ip_address(ec2, stack.name)
output.write('Syncing S3 bucket with the instance...')
# sync S3 with the instance
remote_cmd = subprocess.list2cmdline([
'sudo', '-i', '/bin/bash', '-e', '/tmp/scripts/sync_project.sh',
'>', '/dev/null'
])
# connect to the instance and run remote command
host = 'ubuntu@%s' % ip_address
key_path = KeyPairResource(None, project_name, region).key_path
ssh_command = [
'ssh', '-i', key_path, '-o', 'StrictHostKeyChecking no', '-tq',
host, remote_cmd
]
subprocess.call(ssh_command)
output.write('Done')
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
region = instance_config['region']
availability_zone = instance_config['availabilityZone']
subnet_id = instance_config['subnetId']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
project_name = project_config['name']
stack = StackResource(cf, project_name, region)
# check if the stack already exists
if stack.stack_exists():
raise ValueError('Stack "%s" already exists.\n'
'Use "spotty stop" command to delete the stack.' %
stack.name)
# check availability zone and subnet
check_az_and_subnet(ec2, availability_zone, subnet_id, region)
# sync the project with S3
output.write('Syncing the project with S3...')
sync_filters = project_config['syncFilters']
bucket_name = sync_project_with_s3(self._project_dir, project_name,
region, sync_filters, output)
# create or update instance profile
instance_profile_arn = create_or_update_instance_profile(cf, output)
# prepare CloudFormation template
output.write('Preparing CloudFormation template...')
instance_type = instance_config['instanceType']
volumes = instance_config['volumes']
ports = instance_config['ports']
max_price = instance_config['maxPrice']
docker_commands = instance_config['docker']['commands']
template = stack.prepare_template(ec2, availability_zone, subnet_id,
instance_type, volumes, ports,
max_price, docker_commands)
# create stack
ami_name = instance_config['amiName']
root_volume_size = instance_config['rootVolumeSize']
mount_dirs = [volume['directory'] for volume in volumes]
docker_config = instance_config['docker']
remote_project_dir = project_config['remoteDir']
res = stack.create_stack(ec2, template, instance_profile_arn,
instance_type, ami_name, root_volume_size,
mount_dirs, bucket_name, remote_project_dir,
project_name, self._project_dir,
docker_config)
output.write('Waiting for the stack to be created...')
resource_messages = [
('SpotInstance', 'launching the instance'),
('DockerReadyWaitCondition',
'waiting for the Docker container to be ready'),
]
# wait for the stack to be created
status, info = wait_stack_status_changed(
cf,
stack_id=res['StackId'],
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if status == 'CREATE_COMPLETE':
ip_address = get_instance_ip_address(ec2, stack.name)
availability_zone = [
row['OutputValue'] for row in info['Outputs']
if row['OutputKey'] == 'AvailabilityZone'
][0]
# get the current spot price
current_price = get_current_spot_price(ec2, instance_type,
availability_zone)
output.write(
'\n'
'--------------------\n'
'Instance is running.\n'
'\n'
'IP address: %s\n'
'Current Spot price: $%.04f\n'
'\n'
'Use "spotty ssh" command to connect to the Docker container.\n'
'--------------------' % (ip_address, current_price))
else:
raise ValueError(
'Stack "%s" was not created.\n'
'Please, see CloudFormation and CloudWatch logs for the details.'
% stack.name)