def set_preamble_cmds(self, preamble_cmds):
if not isinstance(preamble_cmds, (tuple, list)):
self.preamble_cmds = [preamble_cmds]
print_err(('[Warning] preamble command "{}" for TmuxExperiment ' +
'"{}" should be a list').format(preamble_cmds, name))
else:
self.preamble_cmds = list(preamble_cmds)
def set_command(self, command):
if not isinstance(command, list):
print_err(
'[Warning] command {} for KubernetesProcess {} must be a list'.
format(command, self.name))
command = [command]
self.container_yml.set_command(command)
def update(self, di):
for key, val in di.items():
if key in self.registry:
setattr(self, key, self.registry[key](val))
else:
print_err(
"[Warning] Key {} unrecognized by symphony config, ignored."
.format(key))
def set_args(self, args):
if not isinstance(args, list):
print_err(
'[Warning] args {} for KubernetesProcess {} should be a list'.
format(args, self.name))
args = [args]
args = list(map(str, args))
self.container_yml.set_args(args)
def action_visit(self, args):
url = self.cluster.external_url(self._get_experiment(args), args.service_name)
if url:
url = 'http://' + url
print(url)
if not args.url_only:
webbrowser.open(url)
else:
print_err('Tensorboard does not yet have an external IP.')
def _interactive_find(self, matches, error_message):
"""
Find partial match of namespace, ask user to verify before switching to
ns or delete experiment.
Used in:
- symphony delete
- symphony ns
Disabled when --force
"""
if len(matches) == 0:
print_err('[Error] {} not found. Please check for typos.'.format(error_message))
sys.exit(1)
elif len(matches) == 1:
match = matches[0]
print_err('[Warning] No exact match. Fuzzy match only finds one candidate: {}'
.format(match))
return match
prompt = '\n'.join(['{}) {}'.format(i, n) for i, n in enumerate(matches)])
prompt = ('Cannot find exact match. Fuzzy matching: \n'
'{}\nEnter your selection 0-{} (enter to select 0, q to quit): '
.format(prompt, len(matches) - 1))
ans = input(prompt)
if not ans.strip(): # blank
ans = '0'
try:
ans = int(ans)
except ValueError: # cannot convert to int, quit
print_err('aborted')
sys.exit(1)
if ans >= len(matches):
print_err('[Error] Must enter a number between 0 - {}'.format(len(matches)-1))
sys.exit(1)
return matches[ans]
def run_verbose(cmd,
print_out=True,
raise_on_error=False,
dry_run=False,
stdin=''):
out, err, retcode = run(cmd, dry_run=dry_run, stdin=stdin)
if retcode != 0:
_print_err_return(out, err, retcode)
msg = 'Command `{}` fails'.format(cmd)
if raise_on_error:
raise RuntimeError(msg)
else:
print_err(msg)
elif out and print_out:
print(out)
return out, err, retcode
def __init__(self, name, cmds=None, start_dir=None):
"""
Args:
name: name of the process
cmds: list of commands to run
start_dir: directory in which the process starts
"""
tmux_name_check(name, 'Process')
super().__init__(name)
if cmds is None:
cmds = []
self.start_dir = os.path.expanduser(start_dir or '.')
if not isinstance(cmds, (tuple, list)):
print_err(
'[Warning] command "{}" for TmuxProcess "{}" should be a list'.
format(cmds, name))
self.cmds = [cmds]
else:
self.cmds = list(cmds)
self.env = {}
def __init__(self, name, start_dir=None, env_name=None, preamble_cmds=None):
"""
Args:
name: name of the Experiment
start_dir: directory where new processes start for this Experiment
preamble_cmds: str or list of str containing commands to run in each
process before the actual command (e.g. `source activate py3`)
"""
tmux_name_check(name, 'ProcessGroup')
super().__init__(name)
self.start_dir = os.path.expanduser(start_dir or '.')
self.env_name = env_name
if preamble_cmds is None:
preamble_cmds = []
if not isinstance(preamble_cmds, (tuple, list)):
self.preamble_cmds = [preamble_cmds]
print_err(('[Warning] preamble command "{}" for TmuxProcessGroup ' +
'"{}" should be a list').format(preamble_cmds, name))
else:
self.preamble_cmds = list(preamble_cmds)
def __init__(self,
name,
node=None,
cmds=None,
allocation=None,
preferred_ports=[],
port_range=None):
"""
Args:
name: name of the process
cmds: list of commands to run
node: Host machine for the process.
This is used to poll for avail. ports and add ssh commands.
"""
tmux_name_check(name, 'Process')
super().__init__(name)
if port_range is None:
port_range = range(6000, 20000)
self.allocation = allocation
self.preferred_ports = preferred_ports
self.port_range = list(preferred_ports) + list(port_range)
self.node = node
if cmds is None:
cmds = []
if not isinstance(cmds, (tuple, list)):
print_err(
'[Warning] command "{}" for TmuxProcess "{}" should be a list'.
format(cmds, name))
self.cmds = [cmds]
else:
self.cmds = list(cmds)
# overwrite CUDA_VISIBLE_DEVICES with set_gpus
self.env = dict(CUDA_VISIBLE_DEVICES='')
self.cpu_cost = None
self.mem_cost = None
self.gpu_compute_cost = None
self.gpu_mem_cost = None
self.hard_placement = None
def _print_err_return(out, err, retcode):
print_err('error code:', retcode)
print_err('*' * 20, 'stderr', '*' * 20)
print_err(err)
print_err('*' * 20, 'stdout', '*' * 20)
print_err(out)
print_err('*' * 46)