def _get_template_parameters(self, instance_profile_arn: str, instance_name: str, bucket_name: str,
sync_filters: list, volumes: List[AbstractInstanceVolume],
container: ContainerDeployment, output: AbstractOutputWriter, dry_run=False):
# get VPC ID
vpc_id = self.get_vpc_id()
# get image info
ami = self._get_ami()
output.write('- AMI: "%s" (%s)' % (ami.name, ami.image_id))
# check root volume size
root_volume_size = self.instance_config.root_volume_size
if root_volume_size and root_volume_size < ami.size:
raise ValueError('Root volume size cannot be less than the size of AMI (%dGB).' % ami.size)
elif not root_volume_size:
# if a root volume size is not specified, make it 5GB larger than the AMI size
root_volume_size = ami.size + 5
# create key pair
key_name = self.key_pair.get_or_create_key(dry_run)
# get mount directories for the volumes
mount_dirs = [volume.mount_dir for volume in volumes]
# get Docker runtime parameters
runtime_parameters = container.get_runtime_parameters(is_gpu_instance(self.instance_config.instance_type))
# print info about the Docker data root
if self.instance_config.docker_data_root:
docker_data_volume_name = [volume.name for volume in volumes
if is_subdir(self.instance_config.docker_data_root, volume.mount_dir)][0]
output.write('- Docker data will be stored on the "%s" volume' % docker_data_volume_name)
# create stack
parameters = {
'VpcId': vpc_id,
'InstanceProfileArn': instance_profile_arn,
'InstanceType': self.instance_config.instance_type,
'KeyName': key_name,
'ImageId': ami.image_id,
'RootVolumeSize': str(root_volume_size),
'VolumeMountDirectories': ('"%s"' % '" "'.join(mount_dirs)) if mount_dirs else '',
'DockerDataRootDirectory': self.instance_config.docker_data_root,
'DockerImage': container.config.image,
'DockerfilePath': container.dockerfile_path,
'DockerBuildContextPath': container.docker_context_path,
'DockerRuntimeParameters': runtime_parameters,
'DockerWorkingDirectory': container.config.working_dir,
'InstanceNameTag': self.ec2_instance_name,
'ProjectS3Path': get_project_s3_path(bucket_name),
'HostProjectDirectory': container.host_project_dir,
'SyncCommandArgs': list2cmdline(get_instance_sync_arguments(sync_filters)),
'UploadS3Path': get_tmp_instance_s3_path(bucket_name, instance_name),
}
return parameters
def _get_host_project_dir(self, volume_mounts: List[VolumeMount]) -> str:
"""Returns the host project directory."""
host_project_dir = None
for volume_mount in sorted(volume_mounts, key=lambda x: len(x.mount_path), reverse=True):
if is_subdir(self.container_config.project_dir, volume_mount.mount_path):
# the project directory is a subdirectory of a Volume Mount directory
project_subdir = os.path.relpath(self.container_config.project_dir, volume_mount.mount_path)
host_project_dir = os.path.normpath(volume_mount.host_path + '/' + project_subdir)
break
# this should not be the case as the volume mount for the project directory should be added automatically
# if it doesn't exist in the configuration
assert host_project_dir is not None, 'A volume mount that contains the project directory not found.'
return host_project_dir
def get_template_parameters(ec2, instance_config: InstanceConfig,
instance_profile_arn: str, bucket_name: str,
key_pair_name: str, output: AbstractOutputWriter):
# get AMI
ami = get_ami(ec2, instance_config.ami_id, instance_config.ami_name)
output.write('- AMI: "%s" (%s)' % (ami.name, ami.image_id))
# check root volume size
root_volume_size = instance_config.root_volume_size
if root_volume_size and root_volume_size < ami.size:
raise ValueError(
'Root volume size cannot be less than the size of AMI (%dGB).' %
ami.size)
elif not root_volume_size:
# if a root volume size is not specified, make it 5GB larger than the AMI size
root_volume_size = ami.size + 5
# print info about the Docker data root
ebs_volumes = [
volume for volume in instance_config.volumes
if isinstance(volume, EbsVolume)
]
if instance_config.docker_data_root:
docker_data_volume_name = [
volume.name for volume in ebs_volumes
if is_subdir(instance_config.docker_data_root, volume.mount_dir)
][0]
output.write('- Docker data will be stored on the "%s" volume' %
docker_data_volume_name)
# create stack
parameters = {
'VpcId': get_vpc_id(ec2, instance_config.subnet_id),
'InstanceProfileArn': instance_profile_arn,
'InstanceType': instance_config.instance_type,
'KeyName': key_pair_name,
'ImageId': ami.image_id,
'RootVolumeSize': str(root_volume_size),
'DockerDataRootDirectory': instance_config.docker_data_root,
'InstanceNameTag': instance_config.ec2_instance_name,
'HostProjectDirectory': instance_config.host_project_dir,
'LogsS3Path': get_logs_s3_path(bucket_name, instance_config.name),
}
return parameters
def _get_volume_mounts(self):
"""Returns container volume mounts from the configuration and
adds the project volume mount if necessary."""
volume_mounts = self._config['volumeMounts']
# check if the project directory is a sub-directory of one of the volume mounts
project_has_volume = False
for volume_mount in volume_mounts:
if is_subdir(self.project_dir, volume_mount['mountPath']):
project_has_volume = True
break
# if it's not, then add new volume mount
if not project_has_volume:
volume_mounts.insert(0, {
'name': PROJECT_VOLUME_MOUNT_NAME,
'mountPath': self.project_dir,
})
return volume_mounts
def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]):
"""Get container volume mounts."""
# get mount directories for the volumes
mount_dirs = OrderedDict([(volume.name, volume.mount_dir)
for volume in volumes])
# get container volumes mapping
volume_mounts = []
for container_volume in self.config.volume_mounts:
volume_name = container_volume['name']
host_dir = mount_dirs.get(
volume_name, '/tmp/spotty/container/volumes/%s' % volume_name)
volume_mounts.append(
VolumeMount(
name=volume_name,
host_dir=host_dir,
container_dir=container_volume['mountPath'],
))
# get host project directory
host_project_dir = None
for name, host_dir, container_dir in volume_mounts:
if is_subdir(self.config.project_dir, container_dir):
project_subdir = os.path.relpath(self.config.project_dir,
container_dir)
host_project_dir = host_dir + '/' + project_subdir
break
if not host_project_dir:
# use temporary directory for the project
host_project_dir = '/tmp/spotty/container/volumes/.project'
volume_mounts.append(
VolumeMount(
name=None,
host_dir=host_project_dir,
container_dir=self.config.project_dir,
))
return volume_mounts, host_project_dir
def prepare_instance_template(instance_config: InstanceConfig, docker_commands: DockerCommands, image_link: str,
bucket_name: str, sync_project_cmd: str, public_key_value: str,
service_account_email: str, output: AbstractOutputWriter):
"""Prepares deployment template to run an instance."""
# get disk attachments
disk_attachments, disk_device_names, disk_mount_dirs = \
_get_disk_attachments(instance_config.volumes, instance_config.zone)
# run sync command as a non-root user
if instance_config.container_config.run_as_host_user:
sync_project_cmd = 'sudo -u %s %s' % (instance_config.user, sync_project_cmd)
startup_scripts_templates = [
{
'filename': '01_prepare_instance.sh',
'params': {
'CONTAINER_BASH_SCRIPT_PATH': CONTAINER_BASH_SCRIPT_PATH,
'CONTAINER_BASH_SCRIPT': ContainerBashScript(docker_commands).render(),
'IS_GPU_INSTANCE': bool(instance_config.gpu),
'SSH_USERNAME': instance_config.user,
'SPOTTY_TMP_DIR': INSTANCE_SPOTTY_TMP_DIR,
'CONTAINERS_TMP_DIR': CONTAINERS_TMP_DIR,
},
},
{
'filename': '02_mount_volumes.sh',
'params': {
'DISK_DEVICE_NAMES': ('"%s"' % '" "'.join(disk_device_names)) if disk_device_names else '',
'DISK_MOUNT_DIRS': ('"%s"' % '" "'.join(disk_mount_dirs)) if disk_mount_dirs else '',
'TMP_VOLUME_DIRS': [{'PATH': volume.host_path} for volume in instance_config.volumes
if isinstance(volume, TmpDirVolume)],
},
},
{
'filename': '03_set_docker_root.sh',
'params': {
'DOCKER_DATA_ROOT_DIR': instance_config.docker_data_root,
},
},
{
'filename': '04_sync_project.sh',
'params': {
'HOST_PROJECT_DIR': instance_config.host_project_dir,
'SYNC_PROJECT_CMD': sync_project_cmd,
},
},
{
'filename': '05_run_instance_startup_commands.sh',
'params': {
'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,
'INSTANCE_STARTUP_COMMANDS': instance_config.commands,
},
},
]
# render startup scripts
startup_scripts_content = []
for template in startup_scripts_templates:
with open(os.path.join(os.path.dirname(__file__), 'data', 'startup_scripts', template['filename'])) as f:
content = f.read()
startup_scripts_content.append({
'filename': template['filename'],
'content': chevron.render(content, template['params'])
})
startup_scripts_content.append({
'filename': '06_start_container.sh',
'content': StartContainerScript(docker_commands).render(print_trace=True),
})
# render the main startup script
with open(os.path.join(os.path.dirname(__file__), 'data', 'startup_script.sh.tpl')) as f:
startup_script = f.read()
startup_script = chevron.render(startup_script, {
'MACHINE_NAME': instance_config.machine_name,
'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,
'STARTUP_SCRIPTS': startup_scripts_content,
})
# render the template
with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:
template = f.read()
template = chevron.render(template, {
'SERVICE_ACCOUNT_EMAIL': service_account_email,
'ZONE': instance_config.zone,
'MACHINE_TYPE': instance_config.machine_type,
'SOURCE_IMAGE': image_link,
'BOOT_DISK_SIZE': instance_config.boot_disk_size,
'MACHINE_NAME': instance_config.machine_name,
'PREEMPTIBLE': 'true' if instance_config.is_preemptible_instance else 'false',
'GPU_TYPE': instance_config.gpu['type'] if instance_config.gpu else '',
'GPU_COUNT': instance_config.gpu['count'] if instance_config.gpu else 0,
'DISK_ATTACHMENTS': disk_attachments,
'SSH_USERNAME': instance_config.user,
'PUB_KEY_VALUE': public_key_value,
'PORTS': ', '.join([str(port) for port in set([22] + instance_config.ports)]),
}, partials_dict={
'STARTUP_SCRIPT': startup_script,
})
# print some information about the deployment
output.write('- image URL: ' + '/'.join(image_link.split('/')[-5:]))
output.write('- zone: ' + instance_config.zone)
output.write('- preemptible VM' if instance_config.is_preemptible_instance else '- on-demand VM')
output.write(('- GPUs: %d x %s' % (instance_config.gpu['count'], instance_config.gpu['type']))
if instance_config.gpu else '- no GPUs')
# print name of the volume where Docker data will be stored
if instance_config.docker_data_root:
docker_data_volume_name = [volume.name for volume in instance_config.volumes
if is_subdir(instance_config.docker_data_root, volume.host_path)][0]
output.write('- Docker data will be stored on the "%s" volume' % docker_data_volume_name)
return template
def prepare_instance_template(instance_config: InstanceConfig, container: ContainerDeployment, sync_filters: list,
volumes: List[AbstractInstanceVolume], machine_name: str, image_link: str,
bucket_name: str, public_key_value: str, service_account_email: str,
output: AbstractOutputWriter):
"""Prepares deployment template to run an instance."""
# read and update the template
with open(os.path.join(os.path.dirname(__file__), 'instance', 'template.yaml')) as f:
template = f.read()
# get disk attachments
disk_attachments, disk_device_names, disk_mount_dirs = _get_disk_attachments(volumes, instance_config.zone)
# get Docker runtime parameters
runtime_parameters = container.get_runtime_parameters(bool(instance_config.gpu))
# render startup script
startup_script = open(os.path.join(os.path.dirname(__file__), 'instance', 'cloud_init.yaml'), 'r').read()
startup_script = chevron.render(startup_script, {
'MACHINE_NAME': machine_name,
'ZONE': instance_config.zone,
'DISK_DEVICE_NAMES': ('"%s"' % '" "'.join(disk_device_names)) if disk_device_names else '',
'DISK_MOUNT_DIRS': ('"%s"' % '" "'.join(disk_mount_dirs)) if disk_mount_dirs else '',
'PROJECT_GS_BUCKET': bucket_name,
'BUCKET_SYNC_DIR': BUCKET_SYNC_DIR,
'HOST_PROJECT_DIR': container.host_project_dir,
'SYNC_ARGS': list2cmdline(get_instance_sync_arguments(sync_filters)),
'DOCKER_DATA_ROOT_DIR': instance_config.docker_data_root,
'DOCKER_IMAGE': container.config.image,
'DOCKERFILE_PATH': container.dockerfile_path,
'DOCKER_BUILD_CONTEXT_PATH': container.docker_context_path,
'DOCKER_RUNTIME_PARAMS': runtime_parameters,
'DOCKER_WORKING_DIR': container.config.working_dir,
})
# render the template
parameters = {
'SERVICE_ACCOUNT_EMAIL': service_account_email,
'ZONE': instance_config.zone,
'MACHINE_TYPE': instance_config.machine_type,
'SOURCE_IMAGE': image_link,
'STARTUP_SCRIPT': fix_indents_for_lines(startup_script, template, '{{{STARTUP_SCRIPT}}}'),
'MACHINE_NAME': machine_name,
'PREEMPTIBLE': 'false' if instance_config.on_demand else 'true',
'GPU_TYPE': instance_config.gpu['type'] if instance_config.gpu else '',
'GPU_COUNT': instance_config.gpu['count'] if instance_config.gpu else 0,
'DISK_ATTACHMENTS': disk_attachments,
'PUB_KEY_VALUE': public_key_value,
'PORTS': ', '.join([str(port) for port in set(container.config.ports + [22])]),
}
template = chevron.render(template, parameters)
# print some information about the deployment
output.write('- image URL: ' + '/'.join(image_link.split('/')[-5:]))
output.write('- zone: ' + instance_config.zone)
output.write('- on-demand VM' if instance_config.on_demand else '- preemptible VM')
output.write(('- GPUs: %d x %s' % (instance_config.gpu['count'], instance_config.gpu['type']))
if instance_config.gpu else '- no GPUs')
# print name of the volume where Docker data will be stored
if instance_config.docker_data_root:
docker_data_volume_name = [volume.name for volume in volumes
if is_subdir(instance_config.docker_data_root, volume.mount_dir)][0]
output.write('- Docker data will be stored on the "%s" volume' % docker_data_volume_name)
return template