def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the command is provided
if not args.custom_args:
raise ValueError('Use the double-dash ("--") to split Spotty arguments from the command that should be '
'executed inside the container.')
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
# sync the project with the instance
if not args.no_sync:
try:
instance_manager.sync(output)
except NothingToDoError:
pass
# generate a "docker exec" command
command = shlex_join(args.custom_args)
command = instance_manager.container_commands.exec(command, interactive=args.interactive, tty=args.tty,
user=args.user)
# execute the command on the host OS
exit_code = instance_manager.exec(command, tty=args.tty)
sys.exit(exit_code)
def run(self, image_name: str = None) -> str:
image_name = image_name if image_name else self._instance_config.container_config.image
# prepare "docker run" arguments
args = ['-td'] + self._instance_config.container_config.runtime_parameters
if self._instance_config.container_config.host_network:
args += ['--net=host']
for port in self._instance_config.container_config.ports:
host_port = port['hostPort']
container_port = port['containerPort']
args += ['-p', ('%d:%d' % (host_port, container_port)) if host_port else str(container_port)]
for volume_mount in self._instance_config.volume_mounts:
args += ['-v', '%s:%s:%s' % (volume_mount.host_path, volume_mount.mount_path, volume_mount.mode)]
args += ['-v', '%s:%s:%s' % ('/efs', '/efs', 'rw')]
for env_name, env_value in self._instance_config.container_config.env.items():
args += ['-e', '%s=%s' % (env_name, env_value)]
args += ['--name', self._instance_config.full_container_name]
run_cmd = 'docker run $(nvidia-smi &> /dev/null && echo "--gpus all")'
if self._instance_config.container_config.run_as_host_user:
run_cmd += ' -u $(id -u %s):$(id -g %s) -e HOST_USER_ID=$(id -u %s) -e HOST_GROUP_ID=$(id -g %s)' \
% tuple([self._instance_config.user] * 4)
run_cmd += ' %s %s /bin/sh > /dev/null' % (shlex_join(args), image_name)
return run_cmd
def get_script_command(script_name: str, script_content: str, script_args: list = None,
logging: bool = False) -> str:
"""Encodes a multi-line script into base64 and returns a one-line command
that unpacks the script to a temporary file and runs it."""
# encode the script content to base64
script_base64 = base64.b64encode(script_content.encode('utf-8')).decode('utf-8')
# command to decode the script, save it to a temporary file and run inside the container
script_args = shlex_join(script_args) if script_args else ''
script_cmd = ' && '.join([
'TMPDIR=${TMPDIR%/}',
'TMP_SCRIPT_PATH=$(mktemp ${TMPDIR:-/tmp}/spotty-%s.XXXXXXXX)' % script_name,
'chmod +x $TMP_SCRIPT_PATH',
'echo %s | base64 -d > $TMP_SCRIPT_PATH' % script_base64,
'$TMP_SCRIPT_PATH ' + script_args,
])
# log the command output to a file
if logging:
log_file_path = '/var/log/spotty/run/%s-%d.log' % (script_name, time.time())
script_cmd = get_log_command(script_cmd, log_file_path)
# execute the command with bash
script_cmd = '%s -c %s' % (get_bash_command(), shlex.quote(script_cmd))
return script_cmd
def get_s3_sync_command(from_path: str,
to_path: str,
profile: str = None,
region: str = None,
filters: list = None,
exact_timestamp: bool = False,
delete: bool = False,
quiet: bool = False,
dry_run: bool = False):
"""Builds an "aws s3 sync" command."""
args = ['aws']
if profile:
args += ['--profile', profile]
if region:
args += ['--region', region]
args += ['s3', 'sync', from_path, to_path]
if filters:
for sync_filter in filters:
if ('exclude' in sync_filter and 'include' in sync_filter) \
or ('exclude' not in sync_filter and 'include' not in sync_filter):
raise ValueError('S3 sync filter has wrong format.')
if 'exclude' in sync_filter:
for path in sync_filter['exclude']:
args += ['--exclude', path]
if 'include' in sync_filter:
for path in sync_filter['include']:
args += ['--include', path]
if exact_timestamp:
args.append('--exact-timestamp')
if delete:
args.append('--delete')
if quiet:
args.append('--quiet')
if dry_run:
args.append('--dryrun')
command = shlex_join(args)
return command
def _get_rsync_command(src_path: str,
dst_path: str,
ssh_port: int,
ssh_key_path: str,
filters: List[dict] = None,
mkdir: str = None,
use_sudo: bool = False,
dry_run: bool = False):
sudo_str = 'sudo ' if use_sudo else ''
remote_rsync_cmd = sudo_str + 'rsync'
if mkdir:
remote_rsync_cmd = '%smkdir -p \'%s\' && %s' % (sudo_str, mkdir,
remote_rsync_cmd)
rsync_cmd = 'rsync -av ' \
'--no-owner ' \
'--no-group ' \
'--prune-empty-dirs ' \
'-e "ssh -i \'%s\' -p %d -o StrictHostKeyChecking=no -o ConnectTimeout=10" ' \
'--rsync-path="%s"' \
% (ssh_key_path, ssh_port, remote_rsync_cmd)
if dry_run:
rsync_cmd += ' --dry-run'
if filters:
args = []
for sync_filter in filters:
if 'exclude' in sync_filter:
for path in sync_filter['exclude']:
args += ['--exclude', _fix_filter_path(path)]
if 'include' in sync_filter:
for path in sync_filter['include']:
args += ['--include', _fix_filter_path(path)]
rsync_cmd += ' ' + shlex_join(args)
rsync_cmd += ' %s/ %s' % (src_path.rstrip('/'), dst_path)
return rsync_cmd
def get_rsync_command(from_path: str,
to_path: str,
filters: List[dict] = None,
delete: bool = False,
quiet: bool = False,
dry_run: bool = False):
args = ['gsutil', '-m']
if quiet:
args.append('-q')
args += ['rsync', '-r']
if filters:
if (len(filters) > 1) or (len(filters[0]) > 1) or ('include'
in filters[0]):
raise ValueError(
'At the moment GCP provider supports only one list of exclude filters.'
)
path_regs = []
for path in filters[0]['exclude']:
path = path.replace('/', os.sep) # fix for Windows machines
path_regs.append(fnmatch.translate(path)[4:-3])
filter_regex = '^(%s)$' % '|'.join(path_regs)
args += ['-x', filter_regex]
if delete:
args.append('-d')
if dry_run:
args.append('-n')
args += [from_path, to_path]
return shlex_join(args)