def build_singularity_run_command(
container_command,
image,
volumes=[],
env=[],
working_directory=DEFAULT_WORKING_DIRECTORY,
singularity_cmd=DEFAULT_SINGULARITY_COMMAND,
run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
sudo=DEFAULT_SUDO,
sudo_cmd=DEFAULT_SUDO_COMMAND,
):
command_parts = []
# http://singularity.lbl.gov/docs-environment-metadata
for (key, value) in env:
command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)])
command_parts += _singularity_prefix(
singularity_cmd=singularity_cmd,
sudo=sudo,
sudo_cmd=sudo_cmd,
)
command_parts.append("exec")
for volume in volumes:
command_parts.extend(["-B", shlex_quote(str(volume))])
if working_directory:
command_parts.extend(["--pwd", shlex_quote(working_directory)])
if run_extra_arguments:
command_parts.append(run_extra_arguments)
full_image = image
command_parts.append(shlex_quote(full_image))
command_parts.append(container_command)
return " ".join(command_parts)
def test_item_removed(self):
# Emit item_removed event for an item that is in a playlist
results = self.lib.items(u'path:{0}'.format(shlex_quote(
os.path.join(self.music_dir, 'd', 'e', 'f.mp3'))))
item = results[0]
beets.plugins.send('item_removed', item=item)
# Emit item_removed event for an item that is not in a playlist
results = self.lib.items(u'path:{0}'.format(shlex_quote(
os.path.join(self.music_dir, 'x', 'y', 'z.mp3'))))
item = results[0]
beets.plugins.send('item_removed', item=item)
# Emit cli_exit event
beets.plugins.send('cli_exit', lib=self.lib)
# Check playlist with absolute paths
playlist_path = os.path.join(self.playlist_dir, 'absolute.m3u')
with open(playlist_path, 'r') as f:
lines = [line.strip() for line in f.readlines()]
self.assertEqual(lines, [
os.path.join(self.music_dir, 'a', 'b', 'c.mp3'),
os.path.join(self.music_dir, 'nonexisting.mp3'),
])
# Check playlist with relative paths
playlist_path = os.path.join(self.playlist_dir, 'relative.m3u')
with open(playlist_path, 'r') as f:
lines = [line.strip() for line in f.readlines()]
self.assertEqual(lines, [
os.path.join('a', 'b', 'c.mp3'),
'nonexisting.mp3',
])
def startup_command(self, ctx, **kwds):
"""Return a shell command used to startup this instance.
Among other common planemo kwds, this should respect the
``daemon`` keyword.
"""
daemon = kwds.get("daemon", False)
# TODO: Allow running dockerized Galaxy here instead.
setup_venv_command = setup_venv(ctx, kwds)
run_script = "%s $COMMON_STARTUP_ARGS" % shlex_quote(os.path.join(self.galaxy_root, "run.sh"))
if daemon:
run_script += " --daemon"
self.env["GALAXY_RUN_ALL"] = "1"
else:
run_script += " --server-name %s" % shlex_quote(self.server_name)
server_ini = os.path.join(self.config_directory, "galaxy.ini")
self.env["GALAXY_CONFIG_FILE"] = server_ini
if parse_version(kwds.get('galaxy_python_version') or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
# We need to start under gunicorn
self.env['APP_WEBSERVER'] = 'gunicorn'
self.env['GUNICORN_CMD_ARGS'] = "--bind={host}:{port} --name={server_name}".format(
host=kwds.get('host', '127.0.0.1'),
port=kwds['port'],
server_name=self.server_name,
)
cd_to_galaxy_command = ['cd', self.galaxy_root]
return shell_join(
cd_to_galaxy_command,
setup_venv_command,
setup_common_startup_args(),
run_script,
)
def make_test_dockerfile(self, docker_file):
"""Used to determine if we need to rebuild the image"""
self.setup_lines()
docker_lines = docker_file.docker_lines + [
"RUN echo {0}".format(shlex_quote(self.action))
, "RUN echo {0}".format(" ".join(self.folders))
, "RUN echo {0}".format(shlex_quote(self.default_cmd))
]
return DockerFile(docker_lines=docker_lines, mtime=docker_file.mtime)
def new_cmd(session, name, cmd, mode, logdir, shell):
if isinstance(cmd, (list, tuple)):
cmd = " ".join(shlex_quote(str(v)) for v in cmd)
if mode == 'tmux':
return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd))
elif mode == 'child':
return name, "{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(cmd, logdir, session, name, logdir)
elif mode == 'nohup':
return name, "nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(shell, shlex_quote(cmd), logdir, session, name, logdir)
def wrap_cmd(session, name, cmd):
if isinstance(cmd, list):
cmd = ' '.join(shlex_quote(str(arg)) for arg in cmd)
if args.mode == 'tmux':
return 'tmux send-keys -t {}:{} {} Enter'.format(session, name, shlex_quote(cmd))
elif args.mode == 'child':
return '{} > {}/{}.{}.out 2>&1 & echo kill $! >> {}/kill.sh'.format(
cmd, args.logdir, session, name, args.logdir
)
def make_first_dockerfile(self, docker_file):
"""
Makes the dockerfile for when we don't already have this image
It will just perform the action after the normal docker lines.
"""
self.setup_lines()
docker_lines = docker_file.docker_lines + [
"RUN {0} -c {1}".format(shlex_quote(self.shell), shlex_quote(self.action))
, "CMD {0}".format(self.default_cmd)
]
return DockerFile(docker_lines=docker_lines, mtime=docker_file.mtime)
def process_in_parallel(tag, total_range_size, binary, output_dir):
"""Run the specified binary cfg.NUM_GPUS times in parallel, each time as a
subprocess that uses one GPU. The binary must accept the command line
arguments `--range {start} {end}` that specify a data processing range.
"""
# Snapshot the current cfg state in order to pass to the inference
# subprocesses
cfg_file = os.path.join(output_dir, '{}_range_config.yaml'.format(tag))
with open(cfg_file, 'w') as f:
yaml.dump(cfg, stream=f)
subprocess_env = os.environ.copy()
processes = []
subinds = np.array_split(range(total_range_size), cfg.NUM_GPUS)
for i in range(cfg.NUM_GPUS):
start = subinds[i][0]
end = subinds[i][-1] + 1
subprocess_env['CUDA_VISIBLE_DEVICES'] = str(i)
cmd = '{binary} --range {start} {end} --cfg {cfg_file} NUM_GPUS 1'
cmd = cmd.format(
binary=shlex_quote(binary),
start=int(start),
end=int(end),
cfg_file=shlex_quote(cfg_file)
)
logger.info('{} range command {}: {}'.format(tag, i, cmd))
if i == 0:
subprocess_stdout = subprocess.PIPE
else:
filename = os.path.join(
output_dir, '%s_range_%s_%s.stdout' % (tag, start, end)
)
subprocess_stdout = open(filename, 'w') # NOQA (close below)
p = subprocess.Popen(
cmd,
shell=True,
env=subprocess_env,
stdout=subprocess_stdout,
stderr=subprocess.STDOUT,
bufsize=1
)
processes.append((i, p, start, end, subprocess_stdout))
# Log output from inference processes and collate their results
outputs = []
for i, p, start, end, subprocess_stdout in processes:
log_subprocess_output(i, p, output_dir, tag, start, end)
if isinstance(subprocess_stdout, file): # NOQA (Python 2 for now)
subprocess_stdout.close()
range_file = os.path.join(
output_dir, '%s_range_%s_%s.pkl' % (tag, start, end)
)
range_data = pickle.load(open(range_file))
outputs.append(range_data)
return outputs
def run_on_control_instead(args, sys_argv):
argv = [arg for arg in sys_argv][1:]
argv.remove('--control')
executable = 'commcare-cloud'
branch = getattr(args, 'branch', 'master')
cmd_parts = [
executable, args.env_name, 'ssh', 'control', '-t',
'source ~/init-ansible && git fetch --prune && git checkout {branch} '
'&& git reset --hard origin/{branch} && source ~/init-ansible && {cchq} {cchq_args}'
.format(branch=branch, cchq=executable, cchq_args=' '.join([shlex_quote(arg) for arg in argv]))
]
print_command(' '.join([shlex_quote(part) for part in cmd_parts]))
os.execvp(executable, cmd_parts)
def __add_arg(args, arg, value):
optarg = '--%s' % arg
if isinstance(value, bool):
if value is True:
args.append(optarg)
elif isinstance(value, string_types):
# the = in --optarg=value is usually, but not always, optional
if value.startswith('='):
args.append(shlex_quote(optarg + value))
else:
args.append(optarg)
args.append(shlex_quote(value))
else:
[__add_arg(args, arg, v) for v in value]
def run(self, args, manage_args):
environment = get_environment(args.env_name)
public_vars = environment.public_vars
# the default 'cchq' is redundant with ansible/group_vars/all.yml
cchq_user = public_vars.get('cchq_user', 'cchq')
deploy_env = environment.meta_config.deploy_env
# the paths here are redundant with ansible/group_vars/all.yml
if args.release:
code_dir = '/home/{cchq_user}/www/{deploy_env}/releases/{release}'.format(
cchq_user=cchq_user, deploy_env=deploy_env, release=args.release)
else:
code_dir = '/home/{cchq_user}/www/{deploy_env}/current'.format(
cchq_user=cchq_user, deploy_env=deploy_env)
remote_command = (
'bash -c "cd {code_dir}; python_env/bin/python manage.py {args}"'
.format(
cchq_user=cchq_user,
code_dir=code_dir,
args=' '.join(shlex_quote(arg) for arg in manage_args),
)
)
args.server = args.server or 'django_manage:0'
if args.tmux:
args.remote_command = remote_command
Tmux(self.parser).run(args, [])
else:
ssh_args = ['sudo -u {cchq_user} {remote_command}'.format(
cchq_user=cchq_user,
remote_command=remote_command,
)]
if manage_args and manage_args[0] in ["shell", "dbshell"]:
# force ssh to allocate a pseudo-terminal
ssh_args = ['-t'] + ssh_args
Ssh(self.parser).run(args, ssh_args)
def run(self, args, ssh_args):
environment = get_environment(args.env_name)
public_vars = environment.public_vars
if args.server == '-':
args.server = 'django_manage:0'
# the default 'cchq' is redundant with ansible/group_vars/all.yml
cchq_user = public_vars.get('cchq_user', 'cchq')
# Name tabs like "droberts (2018-04-13)"
window_name_expression = '"`whoami` (`date +%Y-%m-%d`)"'
if args.remote_command:
ssh_args = [
'-t',
r'sudo -iu {cchq_user} tmux attach \; new-window -n {window_name} {remote_command} '
r'|| sudo -iu {cchq_user} tmux new -n {window_name} {remote_command}'
.format(
cchq_user=cchq_user,
remote_command=shlex_quote('{} ; bash'.format(args.remote_command)),
window_name=window_name_expression,
)
] + ssh_args
else:
ssh_args = [
'-t',
'sudo -iu {cchq_user} tmux attach || sudo -iu {cchq_user} tmux new -n {window_name}'
.format(cchq_user=cchq_user, window_name=window_name_expression)
]
Ssh(self.parser).run(args, ssh_args)
def ansible_playbook(environment, playbook, *cmd_args):
if os.path.isabs(playbook):
playbook_path = playbook
else:
playbook_path = os.path.join(ANSIBLE_DIR, '{playbook}'.format(playbook=playbook))
cmd_parts = (
'ansible-playbook',
playbook_path,
'-i', environment.paths.inventory_source,
'-e', '@{}'.format(environment.paths.vault_yml),
'-e', '@{}'.format(environment.paths.public_yml),
'-e', '@{}'.format(environment.paths.generated_yml),
'--diff',
) + get_limit() + cmd_args
public_vars = environment.public_vars
cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth)
if has_arg(unknown_args, '-D', '--diff') or has_arg(unknown_args, '-C', '--check'):
puts(colored.red("Options --diff and --check not allowed. Please remove -D, --diff, -C, --check."))
puts("These ansible-playbook options are managed automatically by commcare-cloud and cannot be set manually.")
return 2 # exit code
ask_vault_pass = public_vars.get('commcare_cloud_use_vault', True)
if ask_vault_pass:
cmd_parts += ('--vault-password-file={}/echo_vault_password.sh'.format(ANSIBLE_DIR),)
cmd_parts_with_common_ssh_args = get_common_ssh_args(environment, use_factory_auth=use_factory_auth)
cmd_parts += cmd_parts_with_common_ssh_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command(cmd)
env_vars = ansible_context.env_vars
if ask_vault_pass:
env_vars['ANSIBLE_VAULT_PASSWORD'] = environment.get_ansible_vault_password()
return subprocess.call(cmd_parts, env=env_vars)
def format_identity_file(ssh_key_file):
"""Render identity file option."""
if ssh_key_file:
safe_key_path = shlex_quote(str(ssh_key_file))
return '-i {}'.format(safe_key_path)
else:
return ''
def _write_load_config_sh(self, environments_dir, quiet):
puts(colored.blue("Let's get you set up to run commcare-cloud."))
if not environments_dir:
environments_dir = self._determine_environments_dir(quiet=quiet)
commcare_cloud_dir = os.path.expanduser("~/.commcare-cloud")
if not os.path.exists(commcare_cloud_dir):
os.makedirs(commcare_cloud_dir)
load_config_file = os.path.expanduser("~/.commcare-cloud/load_config.sh")
if not os.path.exists(load_config_file) or \
ask("Overwrite your ~/.commcare-cloud/load_config.sh?", quiet=quiet):
with open(load_config_file, 'w') as f:
f.write(textwrap.dedent("""
# auto-generated with `manage-commcare-cloud configure`:
export COMMCARE_CLOUD_ENVIRONMENTS={COMMCARE_CLOUD_ENVIRONMENTS}
export PATH=$PATH:{virtualenv_path}
source {PACKAGE_BASE}/.bash_completion
""".format(
COMMCARE_CLOUD_ENVIRONMENTS=shlex_quote(environments_dir),
virtualenv_path=get_virtualenv_bin_path(),
PACKAGE_BASE=PACKAGE_BASE,
)).strip())
puts(colored.blue("Add the following to your ~/.bash_profile:"))
puts(colored.cyan("source ~/.commcare-cloud/load_config.sh"))
puts(colored.blue(
"and then open a new shell. "
"You should be able to run `commcare-cloud` without entering your virtualenv."))
def match(self, name):
dirpath = dirname(name)
if not any([len(i) > 0 for i in [self.ext_list, self.regex_list, self.glob_list]]):
self.glob_list.append("*") # no filters, return all
ext_matches = []
for e in self.ext_list:
ext_matches.extend(self.glob(dirpath, "*." + e))
glob_matches = []
for p in self.glob_list:
glob_matches.extend(self.glob(dirpath, p))
regex_matches = []
if len(self.regex_list) > 0:
allf = self.glob(dirpath, "*")
for r in self.regex_list:
cr = re.compile(r)
for p in allf:
if cr.match(basename(p)) is not None:
regex_matches.append(p)
nodups = list(set(ext_matches + glob_matches + regex_matches))
prefix = basename(name)
if prefix != "":
lf = ListFilter(nodups)
result = lf.match(prefix)
else:
result = nodups
return [shlex_quote(joinpath(dirpath, n)) for n in result]
def _launch_legacy(self, image, env_override, volumes):
"""Legacy launch method for use when the container interface is not enabled
"""
raw_cmd = self.docker_cmd(image, env_override=env_override, volumes=volumes)
log.info("Starting docker container for IE {0} with command [{1}]".format(
self.attr.viz_id,
' '.join([shlex_quote(x) for x in raw_cmd])
))
p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True)
stdout, stderr = p.communicate()
if p.returncode != 0:
log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr))
return None
else:
container_id = stdout.strip()
log.debug("Container id: %s" % container_id)
inspect_data = self.inspect_container(container_id)
port_mappings = self.get_container_port_mapping(inspect_data)
self.attr.docker_hostname = self.get_container_host(inspect_data)
host_port = self._find_port_mapping(port_mappings)[-1]
log.debug("Container host/port: %s:%s", self.attr.docker_hostname, host_port)
# Now we configure our proxy_requst object and we manually specify
# the port to map to and ensure the proxy is available.
self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy(
self.trans,
host=self.attr.docker_hostname,
port=host_port,
proxy_prefix=self.attr.proxy_prefix,
route_name=self.attr.viz_id,
container_ids=[container_id],
)
# These variables then become available for use in templating URLs
self.attr.proxy_url = self.attr.proxy_request['proxy_url']
def setup_lines(self):
"""
Setup convenience lines for copying and waiting for copying
"""
if getattr(self, "_setup_lines", None):
return
self._setup_lines = True
# Make the shared volume name same as this image name so it doesn't change every time
shared_name = self["shared_name"] = self.image_name().replace('/', '__')
# underscored names for our folders
def without_last_slash(val):
while val and val.endswith("/"):
val = val[:-1]
return val
folders_underscored = self["folders_underscored"] = [(shlex_quote(name.replace("_", "__").replace("/", "_")), shlex_quote(without_last_slash(name))) for name in self.folders]
self["move_from_volume"] = " ; ".join(
"echo {0} && rm -rf {0} && mkdir -p $(dirname {0}) && mv /{1}/{2} {0}".format(name, self.shared_name, underscored)
for underscored, name in self.folders_underscored
)
self["move_into_volume"] = " ; ".join(
"echo {0} && mkdir -p {0} && mv {0} /{1}/{2}".format(name, self.shared_name, underscored)
for underscored, name in self.folders_underscored
)
def explore(args):
if not os.path.exists(args.db):
print("Unable to find database `{}`".format(args.db))
return
conn = sqlite3.connect(args.db)
create_results(conn)
call(['sqlite3', '-column', '-header', shlex_quote(args.db)])
def attempt_port_bind(port):
uwsgi_command = [
"uwsgi",
"--http",
"%s:%s" % (host, port),
"--pythonpath",
os.path.join(galaxy_root, "lib"),
"--yaml",
yaml_config_path,
"--module",
"galaxy.webapps.galaxy.buildapp:uwsgi_app_factory()",
"--enable-threads",
"--die-on-term",
]
handle_uwsgi_cli_command = getattr(
config_object, "handle_uwsgi_cli_command", None
)
if handle_uwsgi_cli_command is not None:
handle_uwsgi_cli_command(uwsgi_command)
# we don't want to quote every argument but we don't want to print unquoted ones either, so do this
log.info("Starting uwsgi with command line: %s", ' '.join([shlex_quote(x) for x in uwsgi_command]))
p = subprocess.Popen(
uwsgi_command,
cwd=galaxy_root,
preexec_fn=os.setsid,
)
return UwsgiServerWrapper(
p, name, host, port
)
def publish_release():
with in_tmpexport(commit) as td, VirtualEnv.temporary() as ve:
if opts['dry_run']:
log.info('Creating source distribution, no upload (dry-run)')
subprocess.check_output(
[ve.python, 'setup.py', 'sdist'],
cwd=td,
)
else:
ve.pip_install(td)
args = [ve.python, 'setup.py',
'sdist',
'upload'
]
if opts['sign']:
args.append('-s')
if opts['identity'] is not None:
args.extend(['-i', shlex_quote(opts['identity'])])
log.info('Uploading signed source distribution to PyPI, '
'using key \'{}\''.format(opts['identity']))
else:
log.info('Uploading signed source distribution to PyPI '
'using default identity')
else:
log.info('Uploading unsigned source distribution to PyPI')
log.debug('Running {}'.format(args))
ve.check_output(
args,
cwd=td,
)
def normalize_command(command):
"""Convert `command` to the string representation.
"""
if isinstance(command, list):
command = list(map(assure_unicode, command))
if len(command) == 1 and command[0] != "--":
# This is either a quoted compound shell command or a simple
# one-item command. Pass it as is.
#
# FIXME: This covers the predominant command-line case, but, for
# Python API callers, it means values like ["./script with spaces"]
# requires additional string-like escaping, which is inconsistent
# with the handling of multi-item lists (and subprocess's
# handling). Once we have a way to detect "running from Python API"
# (discussed in gh-2986), update this.
command = command[0]
else:
if command and command[0] == "--":
# Strip disambiguation marker. Note: "running from Python API"
# FIXME from below applies to this too.
command = command[1:]
command = " ".join(shlex_quote(c) for c in command)
else:
command = assure_unicode(command)
return command
def save_new_settings(self, configuration, image):
self.rotate_sentinel_images()
labels = [(self.configuration_label, b64encode(configuration).decode())]
try:
digests = self._get_digests(self.images)
digests_bucket = json.dumps(digests, sort_keys=True)
digests_bucket = b64encode(digests_bucket.encode()).decode()
labels.append((self.digests_label, digests_bucket))
except fabricio.host_errors:
pass
dockerfile = (
'FROM {image}\n'
'LABEL {labels}\n'
).format(
image=image or 'scratch',
labels=' '.join(itertools.starmap('{0}={1}'.format, labels)),
)
build_command = 'echo {dockerfile} | docker build --tag {tag} -'.format(
dockerfile=shlex_quote(dockerfile),
tag=self.current_settings_tag,
)
try:
fabricio.run(build_command)
except fabricio.host_errors as error:
fabricio.log(
'WARNING: {error}'.format(error=error),
output=sys.stderr,
color=colors.red,
)
def build(self):
xunit_report_file = self.xunit_report_file
sd_report_file = self.structured_report_file
cmd = "sh run_tests.sh $COMMON_STARTUP_ARGS --report_file %s" % shlex_quote(self.html_report_file)
if xunit_report_file:
cmd += " --xunit_report_file %s" % shlex_quote(xunit_report_file)
if sd_report_file:
cmd += " --structured_data_report_file %s" % shlex_quote(sd_report_file)
if self.installed:
cmd += ' -installed'
elif self.failed:
sd = StructuredData(sd_report_file)
tests = " ".join(sd.failed_ids)
cmd += " %s" % tests
else:
cmd += ' functional.test_toolbox'
return cmd
def test_path_query_with_nonexisting_playlist(self):
q = u'playlist:{0}'.format(shlex_quote(os.path.join(
self.playlist_dir,
self.playlist_dir,
'nonexisting.m3u',
)))
results = self.lib.items(q)
self.assertEqual(set(results), set())
def exec_fab_command(env_name, *extra_args):
cmd_parts = (
'fab', '-f', FABFILE,
env_name,
) + tuple(extra_args)
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command(cmd)
return subprocess.call(cmd_parts)
def print_command(command):
"""
commcare-cloud commands by convention print the underlying command they execute
Use this function to do so
"""
if isinstance(command, (list, tuple)):
command = ' '.join(shlex_quote(arg) for arg in command)
print(colored.cyan(command), file=sys.stderr)
def ssh_run_cmd(connect_string, cmd, remote_cwd=None, *args, **kwargs):
if remote_cwd:
cmd_real = 'mkdir -p {0} ; cd {0} ; '.format(shlex_quote(remote_cwd))
else:
cmd_real = ''
cmd_real += escape_cmd_for_ssh(cmd)
ssh_cmd = ['ssh', '-q', '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null', connect_string, cmd_real]
return run_cmd(ssh_cmd, *args, **kwargs)
def get_display_text(self, quote=True):
"""
Returns a string containing the value that would be displayed to the user in the tool interface.
When quote is True (default), the string is escaped for e.g. command-line usage.
"""
rval = self.input.value_to_display_text(self.value) or ''
if quote:
return shlex_quote(rval)
return rval
def run(self, args, ssh_args):
address = self.lookup_server_address(args)
if ':' in address:
address, port = address.split(':')
ssh_args = ['-p', port] + ssh_args
cmd_parts = [self.command, address] + ssh_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command(cmd)
return subprocess.call(cmd_parts)
def fmt(val):
if isinstance(val, float):
val = round(val, 4)
return shlex_quote(format_flag_val(val))
def _run_in_container(self, node, *command):
command = ' '.join([shlex_quote(str(arg)) for arg in command])
self._get_container(node).exec_run(command)
def _exec_attr(self):
return "${python_exe} -um guild.op_main %s ${flag_args}" % shlex_quote(
self._script_module()
)
def create_commands(session,
num_workers,
remotes,
env_id,
logdir,
shell='bash',
policy='lstm',
mode='tmux',
visualise=False):
# for launching the TF workers and for launching tensorboard
base_cmd = [
'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir',
logdir, '--env-id', env_id, '--num-workers',
str(num_workers)
]
if visualise:
base_cmd += ['--visualise']
if remotes is None:
remotes = ["1"] * num_workers
else:
remotes = remotes.split(',')
assert len(remotes) == num_workers
cmds_map = [
new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir,
shell)
]
for i in range(num_workers):
cmds_map += [
new_cmd(
session, "w-%d" % i, base_cmd + [
"--job-name", "worker", "--task",
str(i), "--remotes", remotes[i], "--policy", policy
], mode, logdir, shell)
]
cmds_map += [
new_cmd(session, "tb",
["tensorboard", "--logdir", logdir, "--port", "12345"], mode,
logdir, shell)
]
if mode == 'tmux':
cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)]
windows = [v[0] for v in cmds_map]
notes = []
cmds = [
"mkdir -p {}".format(logdir),
"echo {} {} > {}/cmd.sh".format(
sys.executable,
' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']),
logdir),
]
if mode == 'nohup' or mode == 'child':
cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)]
notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)]
if mode == 'tmux':
notes += [
"Use `tmux attach -t {}` to watch process output".format(session)
]
notes += [
"Use `tmux kill-session -t {}` to kill the job".format(session)
]
else:
notes += [
"Use `tail -f {}/*.out` to watch process output".format(logdir)
]
notes += [
"Point your browser to http://localhost:12345 to see Tensorboard"
]
if mode == 'tmux':
cmds += [
"kill $( lsof -i:12345 -t ) > /dev/null 2>&1", # kill any process using tensorboard's port
"kill $( lsof -i:12222-{} -t ) > /dev/null 2>&1".format(
num_workers +
12222), # kill any processes using ps / worker ports
"tmux kill-session -t {}".format(session),
"tmux new-session -s {} -n {} -d {}".format(
session, windows[0], shell)
]
for w in windows[1:]:
cmds += [
"tmux new-window -t {} -n {} {}".format(session, w, shell)
]
cmds += ["sleep 1"]
for window, cmd in cmds_map:
cmds += [cmd]
return cmds, notes
def create_commands(session,
num_workers,
remotes,
logdir,
shell='bash',
visualise=False):
# 启动各个 Worker 和 启动 TensorBoard
base_cmd = [
'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir',
logdir, '--num-workers',
str(num_workers)
]
# 如果需要可视化
if visualise:
base_cmd += ['--visualise']
if remotes is None:
remotes = ["1"] * num_workers
else:
remotes = remotes.split(',')
assert len(remotes) == num_workers
cmds_map = [
new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], logdir, shell)
]
# 对每一个 worker
for i in range(num_workers):
cmds_map += [
new_cmd(
session, "w-%d" % i, base_cmd + [
"--job-name", "worker", "--task",
str(i), "--remotes", remotes[i]
], logdir, shell)
]
cmds_map += [
new_cmd(session, "tb",
["tensorboard", "--logdir", logdir, "--port", "6006"], logdir,
shell)
]
notes = []
cmds = [
"mkdir -p {}".format(logdir),
"echo {} {} > {}/cmd.sh".format(
sys.executable,
' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']),
logdir),
]
cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)]
notes += ["运行 `source {}/kill.sh` 命令来结束各个进程".format(logdir)]
notes += ["运行 `tail -f {}/*.out` 命令来查看各个进程的输出".format(logdir)]
notes += ["在浏览器中打开 http://localhost:6006 ，查看 TensorBoard 运行的结果"]
for window, cmd in cmds_map:
cmds += [cmd]
return cmds, notes
def create_commands(session, num_workers, log_dir, shell='bash'):
# for launching the TF workers and for launching tensorboard
base_cmd = ['CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py']
cmds_map = [
new_cmd(
session, "ps",
base_cmd + ["--job-name", "ps", "--num-workers",
str(num_workers)])
]
for i in range(num_workers):
cmds_map += [
new_cmd(
session, "w-%d" % i, base_cmd + [
"--job-name", "worker", "--task",
str(i), "--num-workers",
str(num_workers)
])
]
cmds_map += [
new_cmd(session, "tb", [
"tensorboard", "--logdir", log_dir, "--port", "{}".format(TB_PORT)
])
]
cmds_map += [new_cmd(session, "htop", ["htop"])]
windows = [v[0] for v in cmds_map]
notes = []
cmds = [
"mkdir -p {}".format(log_dir),
"echo {} {} > {}/cmd.sh".format(
sys.executable,
' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']),
log_dir),
]
notes += [
"Use `tmux attach -t {}` to watch process output".format(session)
]
notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)]
notes += [
"Point your browser to http://localhost:{} to see Tensorboard".format(
TB_PORT)
]
cmds += [
"kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(
TB_PORT), # kill any process using tensorboard's port
"kill $( lsof -i:{}-{} -t ) > /dev/null 2>&1".format(
PS_PORT, num_workers +
PS_PORT), # kill any processes using ps / worker ports
"tmux kill-session -t {}".format(session),
"tmux new-session -s {} -n {} -d {}".format(session, windows[0], shell)
]
for w in windows[1:]:
cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)]
cmds += ["sleep 1"]
for window, cmd in cmds_map:
cmds += [cmd]
return cmds, notes
def test_flags_reqs_f(d):
p = get_exists_path('libsimpleflag')
reqs_file = d.write_to('reqs', [shlex_quote(p) + ' -DCGET_FLAG=On'])
d.cmds(install_cmds(url='-f {}'.format(reqs_file), alias=p))
def run_ansible_module(environment,
ansible_context,
inventory_group,
module,
module_args,
become=True,
become_user=None,
use_factory_auth=False,
quiet=False,
extra_args=()):
extra_args = tuple(extra_args)
if not quiet:
extra_args = ("--diff", ) + extra_args
else:
extra_args = ("--one-line", ) + extra_args
cmd_parts = (
'ansible',
inventory_group,
'-m',
module,
'-i',
environment.paths.inventory_source,
'-a',
module_args,
) + extra_args
environment.create_generated_yml()
public_vars = environment.public_vars
cmd_parts += get_user_arg(public_vars,
extra_args,
use_factory_auth=use_factory_auth)
become = become or bool(become_user)
become_user = become_user
needs_secrets = False
env_vars = ansible_context.env_vars
if become:
cmd_parts += ('--become', )
needs_secrets = True
if become_user:
cmd_parts += ('--become-user', become_user)
if needs_secrets:
cmd_parts += (
'-e',
'@{}'.format(environment.paths.public_yml),
'-e',
'@{}'.format(environment.paths.generated_yml),
)
cmd_parts += environment.secrets_backend.get_extra_ansible_args()
env_vars.update(
environment.secrets_backend.get_extra_ansible_env_vars())
cmd_parts_with_common_ssh_args = get_common_ssh_args(
environment, use_factory_auth=use_factory_auth)
cmd_parts += cmd_parts_with_common_ssh_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
if not quiet:
print_command(cmd)
return subprocess.call(cmd_parts, env=env_vars)
def build_docker_run_command(
container_command,
image,
interactive=False,
terminal=False,
tag=None,
volumes=[],
volumes_from=DEFAULT_VOLUMES_FROM,
memory=DEFAULT_MEMORY,
env_directives=[],
working_directory=DEFAULT_WORKING_DIRECTORY,
name=None,
net=DEFAULT_NET,
run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
docker_cmd=DEFAULT_DOCKER_COMMAND,
sudo=DEFAULT_SUDO,
sudo_cmd=DEFAULT_SUDO_COMMAND,
auto_rm=DEFAULT_AUTO_REMOVE,
set_user=DEFAULT_SET_USER,
host=DEFAULT_HOST,
):
command_parts = _docker_prefix(
docker_cmd=docker_cmd,
sudo=sudo,
sudo_cmd=sudo_cmd,
host=host
)
command_parts.append("run")
if interactive:
command_parts.append("-i")
if terminal:
command_parts.append("-t")
for env_directive in env_directives:
# e.g. -e "GALAXY_SLOTS=$GALAXY_SLOTS"
# These are environment variable expansions so we don't quote these.
command_parts.extend(["-e", env_directive])
for volume in volumes:
# These are environment variable expansions so we don't quote these.
volume_str = str(volume)
if "$" not in volume_str:
volume_for_cmd_line = shlex_quote(volume_str)
else:
# e.g. $_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw so don't single quote.
volume_for_cmd_line = '"%s"' % volume_str
command_parts.extend(["-v", volume_for_cmd_line])
if volumes_from:
command_parts.extend(["--volumes-from", shlex_quote(str(volumes_from))])
if memory:
command_parts.extend(["-m", shlex_quote(memory)])
if name:
command_parts.extend(["--name", shlex_quote(name)])
if working_directory:
command_parts.extend(["-w", shlex_quote(working_directory)])
if net:
command_parts.extend(["--net", shlex_quote(net)])
if auto_rm:
command_parts.append("--rm")
if run_extra_arguments:
command_parts.append(run_extra_arguments)
if set_user:
user = set_user
if set_user == DEFAULT_SET_USER:
# If future-us is ever in here and fixing this for docker-machine just
# use cwltool.docker_id - it takes care of this default nicely.
euid = os.geteuid()
egid = os.getgid()
user = "******" % (euid, egid)
command_parts.extend(["--user", user])
full_image = image
if tag:
full_image = "%s:%s" % (full_image, tag)
command_parts.append(shlex_quote(full_image))
command_parts.append(container_command)
return " ".join(command_parts)
def git_log(cfg, filenames, rev_range=None):
'''
A git log implementation that allows more flexibility:
- Follow multiple files
- Exclude commits we don't want to see
'''
if len(filenames) == 0:
print("Specify at least one file to log")
elif len(filenames) == 1 and not cfg.excluded_commits:
if rev_range:
cmd = 'git log --follow -p %s %s' % (rev_range, filenames[0])
else:
cmd = 'git log --follow -p %s' % (filenames[0])
os.system(cmd)
else:
# To show renames properly, we have to switch to the root
# of the repository and then specify the potentially renamed file
# for each commit
oldcwd = os.getcwd()
git_toplevel = str(sh.git('rev-parse', '--show-toplevel')).strip()
# Use git log to generate lists of commits for each file, sort
commit_data = []
for fname in filenames:
commit_data += _get_commits(fname, rev_range)
if not len(commit_data):
return
# Sort the results by timestamp
if len(filenames) > 1:
commit_data.sort(reverse=True)
# Create an index of filenames per commit id
fname_by_commit = {}
for _, commit, fname in commit_data:
fname_by_commit.setdefault(commit, []).append(fname)
# Uniquify (http://www.peterbe.com/plog/uniqifiers-benchmark)
seen = set()
seen_add = seen.add
commits = [
c for _, c, _ in commit_data if not (c in seen or seen_add(c))
]
# Finally, display them
try:
os.chdir(git_toplevel)
commands = []
for commit in commits:
if not cfg.is_commit_excluded(commit):
for i, fname in enumerate(sorted(fname_by_commit[commit])):
fname = shlex_quote(fname)
# git log --follow only allows a single filename, so
# merge the outputs together using separate commands
if i == 0:
commands.append(
'git log -p -1 --follow --color %s -- %s' %
(commit, fname))
else:
commands.append(
'git log -p -1 --follow --color --pretty='
' %s -- %s' % (commit, fname))
_multi_output(commands)
finally:
os.chdir(oldcwd)
def save(self, *args, **kwargs): # pylint: disable=arguments-differ
self.value = shlex_quote(self.value)
return super().save(*args, **kwargs)
def git_diff(filenames, rev_range):
commands = [
'git diff --follow -w --color %s %s' % (rev_range, shlex_quote(f))
for f in filenames
]
_multi_output(commands)
def create_commands(session, args, shell='bash'):
# for launching the TF workers and for launching tensorboard
base_cmd = [
sys.executable,
'worker.py',
'--log',
args.log,
'--env-id',
args.env_id,
'--num-workers',
str(args.num_workers),
'--num-ps',
str(args.num_ps),
'--alg',
args.alg,
'--model',
args.model,
'--max-step',
args.max_step,
'--t-max',
args.t_max,
'--eps-step',
args.eps_step,
'--eps',
args.eps,
'--eps-eval',
args.eps_eval,
'--gamma',
args.gamma,
'--lr',
args.lr,
'--decay',
args.decay,
'--sync',
args.sync,
'--update-freq',
args.update_freq,
'--eval-freq',
args.eval_freq,
'--eval-num',
args.eval_num,
'--prediction-step',
args.prediction_step,
'--dim',
args.dim,
'--f-num',
args.f_num,
'--f-pad',
args.f_pad,
'--f-stride',
args.f_stride,
'--f-size',
args.f_size,
'--h-dim',
args.h_dim,
'--branch',
args.branch,
'--config',
args.config,
'--buf',
args.buf,
]
if len(args.gpu) > 0:
base_cmd += ['--gpu', 1]
if args.remotes is None:
args.remotes = ["1"] * args.num_workers
else:
args.remotes = args.remotes.split(',')
assert len(args.remotes) == args.num_workers
cmds_map = []
for i in range(args.num_ps + 1):
prefix = ['CUDA_VISIBLE_DEVICES=']
cmds_map += [
new_cmd(session, "ps-%d" % i,
prefix + base_cmd + ["--job-name", "ps", "--task",
str(i)], args.mode, args.log, shell)
]
for i in range(args.num_workers):
prefix = []
if len(args.gpu) > 0:
prefix = [
'CUDA_VISIBLE_DEVICES=%d' % args.gpu[(i % len(args.gpu))]
]
else:
prefix = ['CUDA_VISIBLE_DEVICES=']
cmds_map += [
new_cmd(
session, "w-%d" % i, prefix + base_cmd + [
"--job-name", "worker", "--task",
str(i), "--remotes", args.remotes[i]
], args.mode, args.log, shell)
]
cmds_map += [
new_cmd(session, "tb",
["tensorboard", "--logdir", args.log, "--port", "12345"],
args.mode, args.log, shell)
]
cmds_map += [
new_cmd(
session, "test", prefix + base_cmd +
["--job-name", "test", "--task",
str(args.num_workers)], args.mode, args.log, shell)
]
windows = [v[0] for v in cmds_map]
notes = []
cmds = [
"mkdir -p {}".format(args.log),
"echo {} {} > {}/cmd.sh".format(
sys.executable,
' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']),
args.log),
]
if args.mode == 'nohup' or args.mode == 'child':
cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(args.log)]
notes += ["Run `source {}/kill.sh` to kill the job".format(args.log)]
if args.mode == 'tmux':
notes += [
"Use `tmux attach -t {}` to watch process output".format(session)
]
notes += [
"Use `tmux kill-session -t {}` to kill the job".format(session)
]
else:
notes += [
"Use `tail -f {}/*.out` to watch process output".format(args.log)
]
notes += [
"Point your browser to http://localhost:12345 to see Tensorboard"
]
if args.mode == 'tmux':
cmds += [
"tmux kill-session -t {}".format(session),
"tmux new-session -s {} -n {} -d {}".format(
session, windows[0], shell)
]
for w in windows[1:]:
cmds += [
"tmux new-window -t {} -n {} {}".format(session, w, shell)
]
cmds += ["sleep 1"]
for window, cmd in cmds_map:
cmds += [cmd]
return cmds, notes
def mull_targets(
targets,
involucro_context=None,
command="build",
channels=DEFAULT_CHANNELS,
namespace="biocontainers",
test='true',
test_files=None,
image_build=None,
name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE,
dry_run=False,
conda_version=None,
verbose=False,
binds=DEFAULT_BINDS,
rebuild=True,
oauth_token=None,
hash_func="v2",
singularity=False,
singularity_image_dir="singularity_import",
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace":
namespace,
"image":
image_function(targets,
image_build=image_build,
name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(
repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(
image_name_parts
) == 2, ": not allowed in image name [%s]" % repo_template_kwds[
"image"]
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name,
oauth_token)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel.lstrip('file://')
binds.append('/%s:/%s' % (bind_path, bind_path))
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f',
'%s/invfile.lua' % DIRNAME,
'-set',
"CHANNELS='%s'" % channels,
'-set',
"TARGETS='%s'" % target_str,
'-set',
"REPO='%s'" % repo,
'-set',
"BINDS='%s'" % bind_str,
]
if DEST_BASE_IMAGE:
involucro_args.extend(
["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
if CONDA_IMAGE:
involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE])
if verbose:
involucro_args.extend(["-set", "VERBOSE='1'"])
if singularity:
singularity_image_name = repo_template_kwds['image']
involucro_args.extend(["-set", "SINGULARITY='1'"])
involucro_args.extend(
["-set",
"SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name])
involucro_args.extend(
["-set",
"SINGULARITY_IMAGE_DIR='%s'" % singularity_image_dir])
involucro_args.extend(
["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid())])
if test:
involucro_args.extend(["-set", "TEST=%s" % shlex_quote(test)])
if conda_version is not None:
verbose = "--verbose" if verbose else "--quiet"
involucro_args.extend([
"-set",
"PREINSTALL='conda install %s --yes conda=%s'" %
(verbose, conda_version)
])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ':' not in test_file:
if os.path.exists(test_file):
test_bind.append(
"%s:%s/%s" %
(test_file, DEFAULT_WORKING_DIR, test_file))
else:
if os.path.exists(test_file.split(':')[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, '-set')
involucro_args.insert(7, "TEST_BINDS='%s'" % ",".join(test_bind))
print(" ".join(involucro_context.build_command(involucro_args)))
if not dry_run:
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity'),
'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {'container_test': test}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
return 0
def shlex_join(strings):
return ' '.join(shlex_quote(s) for s in strings)
def get_compile_command(click_ctx):
"""
Returns a normalized compile command depending on cli context.
The command will be normalized by:
- expanding options short to long
- removing values that are already default
- sorting the arguments
- removing one-off arguments like '--upgrade'
- removing arguments that don't change build behaviour like '--verbose'
"""
from piptools.scripts.compile import cli
# Map of the compile cli options (option name -> click.Option)
compile_options = {option.name: option for option in cli.params}
left_args = []
right_args = []
for option_name, value in click_ctx.params.items():
option = compile_options[option_name]
# Get the latest option name (usually it'll be a long name)
option_long_name = option.opts[-1]
# Collect variadic args separately, they will be added
# at the end of the command later
if option.nargs < 0:
right_args.extend([shlex_quote(force_text(val)) for val in value])
continue
# Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
# or options that don't change compile behaviour (--verbose/--dry-run/...)
if option_long_name in COMPILE_EXCLUDE_OPTIONS:
continue
# Skip options without a value
if option.default is None and not value:
continue
# Skip options with a default value
if option.default == value:
continue
# Use a file name for file-like objects
if isinstance(value, LazyFile):
value = value.name
# Convert value to the list
if not isinstance(value, (tuple, list)):
value = [value]
for val in value:
# Flags don't have a value, thus add to args true or false option long name
if option.is_flag:
# If there are false-options, choose an option name depending on a value
if option.secondary_opts:
# Get the latest false-option
secondary_option_long_name = option.secondary_opts[-1]
arg = option_long_name if val else secondary_option_long_name
# There are no false-options, use true-option
else:
arg = option_long_name
left_args.append(shlex_quote(arg))
# Append to args the option with a value
else:
left_args.append(
"{option}={value}".format(
option=option_long_name, value=shlex_quote(force_text(val))
)
)
return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))
def new_cmd(session, name, cmd):
if isinstance(cmd, (list, tuple)):
cmd = " ".join(shlex_quote(str(v)) for v in cmd)
return name, "tmux send-keys -t {}:{} {} Enter".format(
session, name, shlex_quote(cmd))
def _stringify_kwopt_string(self, flag, val):
"""
"""
return '{flag} {value}'.format(flag=flag, value=shlex_quote(str(val)))
def test_reqs_f(d):
reqs_file = d.write_to('reqs', [shlex_quote(get_exists_path('libsimple'))])
d.cmds(
install_cmds(url='-f {}'.format(reqs_file),
lib='simple',
alias=get_exists_path('libsimple')))
def _stringify_kwopt_list(self, flag, val):
"""
"""
if isinstance(val, string_types):
return self._stringify_kwopt_string(flag, val)
return ' '.join(['{flag} {value}'.format(flag=flag, value=shlex_quote(str(v))) for v in val])
def test_comments_reqs_f(d):
p = get_exists_path('libsimple')
reqs_file = d.write_to(
'reqs', [shlex_quote(p) + ' #A comment', '# Another comment'])
d.cmds(install_cmds(url='-f {}'.format(reqs_file), alias=p))
def build_docker_run_command(
container_command,
image,
interactive=False,
terminal=False,
tag=None,
volumes=[],
volumes_from=DEFAULT_VOLUMES_FROM,
memory=DEFAULT_MEMORY,
env_directives=[],
working_directory=DEFAULT_WORKING_DIRECTORY,
name=None,
net=DEFAULT_NET,
run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
docker_cmd=DEFAULT_DOCKER_COMMAND,
sudo=DEFAULT_SUDO,
sudo_cmd=DEFAULT_SUDO_COMMAND,
auto_rm=DEFAULT_AUTO_REMOVE,
set_user=DEFAULT_SET_USER,
host=DEFAULT_HOST,
):
command_parts = _docker_prefix(docker_cmd=docker_cmd,
sudo=sudo,
sudo_cmd=sudo_cmd,
host=host)
command_parts.append("run")
if interactive:
command_parts.append("-i")
if terminal:
command_parts.append("-t")
for env_directive in env_directives:
command_parts.extend(["-e", shlex_quote(env_directive)])
for volume in volumes:
command_parts.extend(["-v", shlex_quote(str(volume))])
if volumes_from:
command_parts.extend(
["--volumes-from",
shlex_quote(str(volumes_from))])
if memory:
command_parts.extend(["-m", shlex_quote(memory)])
if name:
command_parts.extend(["--name", shlex_quote(name)])
if working_directory:
command_parts.extend(["-w", shlex_quote(working_directory)])
if net:
command_parts.extend(["--net", shlex_quote(net)])
if auto_rm:
command_parts.append("--rm")
if run_extra_arguments:
command_parts.append(run_extra_arguments)
if set_user:
user = set_user
if set_user == DEFAULT_SET_USER:
user = str(os.geteuid())
command_parts.extend(["-u", user])
full_image = image
if tag:
full_image = "%s:%s" % (full_image, tag)
command_parts.append(shlex_quote(full_image))
command_parts.append(container_command)
return " ".join(command_parts)
def run(cmd, show_cmd=False, stdout=False, logfile=None, can_fail=False, workdir=None, stdin_data=None, return_stdout=True, buffer_size=4096, **kwargs):
"""Run a command in shell.
@param show_cmd: show command in stdout/log
@type show_cmd: bool
@param stdout: print output to stdout
@type stdout: bool
@param logfile: save output to logfile
@type logfile: str
@param can_fail: when set, retcode is returned instead of raising RuntimeError
@type can_fail: bool
@param workdir: change current directory to workdir before starting a command
@type workdir: str
@param stdin_data: stdin data passed to a command
@type stdin_data: str
@param buffer_size: size of buffer for reading from proc's stdout, use -1 for line-buffering
@type buffer_size: int
@return_stdout: return command stdout as a function result (turn off when working with large data, None is returned instead of stdout)
@return_stdout: bool
@return: (command return code, merged stdout+stderr)
@rtype: (int, str) or (int, None)
"""
if type(cmd) in (list, tuple):
cmd = " ".join((shlex_quote(i) for i in cmd))
universal_newlines = kwargs.get('universal_newlines', False)
log = None
if logfile:
logfile = os.path.join(workdir or "", logfile)
# What happens to log file if it exists already? If show_cmd is True,
# it will be overwritten. Otherwise the command output will just be
# appended to the existing file.
mode = 'a' if not show_cmd and os.path.exists(logfile) else 'w'
if not universal_newlines:
mode += 'b'
log = open(logfile, mode)
try:
if show_cmd:
command = "COMMAND: %s\n%s\n" % (
cmd,
"-" * min(len(cmd) + 9, 79)
)
if stdout:
print(command, end='')
if logfile:
if six.PY3 and not universal_newlines:
# Log file opened as binary, encode the command
command = bytes(command, encoding='utf-8')
log.write(command)
stdin = None
if stdin_data is not None:
stdin = subprocess.PIPE
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stdin=stdin,
cwd=workdir, **kwargs)
if stdin_data is not None:
class StdinThread(threading.Thread):
def run(self):
proc.stdin.write(stdin_data)
proc.stdin.close()
stdin_thread = StdinThread()
stdin_thread.daemon = True
stdin_thread.start()
output = "" if universal_newlines else b""
sentinel = "" if universal_newlines else b""
while True:
if buffer_size == -1:
lines = proc.stdout.readline()
else:
try:
lines = proc.stdout.read(buffer_size)
except (IOError, OSError) as ex:
import errno
if ex.errno == errno.EINTR:
continue
else:
raise
if lines == sentinel:
break
if stdout:
if not universal_newlines:
sys.stdout.write(lines.decode('utf-8'))
else:
sys.stdout.write(lines)
if logfile:
log.write(lines)
if return_stdout:
output += lines
proc.wait()
finally:
if logfile:
log.close()
if stdin_data is not None:
stdin_thread.join()
err_msg = "ERROR running command: %s" % cmd
if logfile:
err_msg += "\nFor more details see %s" % logfile
if proc.returncode != 0 and not can_fail:
exc = RuntimeError(err_msg)
exc.output = output
raise exc
if proc.returncode != 0 and show_cmd:
print(err_msg, file=sys.stderr)
if not return_stdout:
output = None
return (proc.returncode, output)
def process_in_parallel(tag,
total_range_size,
binary,
output_dir,
load_ckpt,
load_detectron,
opts=''):
"""Run the specified binary NUM_GPUS times in parallel, each time as a
subprocess that uses one GPU. The binary must accept the command line
arguments `--range {start} {end}` that specify a data processing range.
"""
# Snapshot the current cfg state in order to pass to the inference
# subprocesses
cfg_file = os.path.join(output_dir, '{}_range_config.yaml'.format(tag))
with open(cfg_file, 'w') as f:
yaml.dump(cfg, stream=f)
subprocess_env = os.environ.copy()
processes = []
NUM_GPUS = torch.cuda.device_count()
subinds = np.array_split(range(total_range_size), NUM_GPUS)
# Determine GPUs to use
cuda_visible_devices = os.environ.get('CUDA_VISIBLE_DEVICES')
if cuda_visible_devices:
gpu_inds = list(map(int, cuda_visible_devices.split(',')))
assert -1 not in gpu_inds, \
'Hiding GPU indices using the \'-1\' index is not supported'
else:
gpu_inds = range(cfg.NUM_GPUS)
gpu_inds = list(gpu_inds)
# Run the binary in cfg.NUM_GPUS subprocesses
for i, gpu_ind in enumerate(gpu_inds):
start = subinds[i][0]
# pdb.set_trace()
end = subinds[i][-1] + 1
subprocess_env['CUDA_VISIBLE_DEVICES'] = str(gpu_ind)
cmd = (
'python3 {binary} --range {start} {end} --cfg {cfg_file} --set {opts} '
'--output_dir {output_dir}')
if load_ckpt is not None:
cmd += ' --load_ckpt {load_ckpt}'
elif load_detectron is not None:
cmd += ' --load_detectron {load_detectron}'
cmd = cmd.format(binary=shlex_quote(binary),
start=int(start),
end=int(end),
cfg_file=shlex_quote(cfg_file),
output_dir=output_dir,
load_ckpt=load_ckpt,
load_detectron=load_detectron,
opts=' '.join([shlex_quote(opt) for opt in opts]))
logger.info('{} range command {}: {}'.format(tag, i, cmd))
if i == 0:
subprocess_stdout = subprocess.PIPE
else:
filename = os.path.join(
output_dir, '%s_range_%s_%s.stdout' % (tag, start, end))
subprocess_stdout = open(filename, 'w')
p = subprocess.Popen(cmd,
shell=True,
env=subprocess_env,
stdout=subprocess_stdout,
stderr=subprocess.STDOUT,
bufsize=1)
# pdb.set_trace()
processes.append((i, p, start, end, subprocess_stdout))
# Log output from inference processes and collate their results
outputs = []
for i, p, start, end, subprocess_stdout in processes:
log_subprocess_output(i, p, output_dir, tag, start, end)
if isinstance(subprocess_stdout, IOBase):
subprocess_stdout.close()
range_file = os.path.join(output_dir,
'%s_range_%s_%s.pkl' % (tag, start, end))
range_data = pickle.load(open(range_file, 'rb'))
outputs.append(range_data)
return outputs
def run(self, args, manage_args):
environment = get_environment(args.env_name)
public_vars = environment.public_vars
# the default 'cchq' is redundant with ansible/group_vars/all.yml
cchq_user = public_vars.get('cchq_user', 'cchq')
deploy_env = environment.meta_config.deploy_env
# the paths here are redundant with ansible/group_vars/all.yml
if args.release:
code_dir = '/home/{cchq_user}/www/{deploy_env}/releases/{release}'.format(
cchq_user=cchq_user,
deploy_env=deploy_env,
release=args.release)
else:
code_dir = '/home/{cchq_user}/www/{deploy_env}/current'.format(
cchq_user=cchq_user, deploy_env=deploy_env)
def _get_ssh_args(remote_command):
return [
'sudo -iu {cchq_user} bash -c {remote_command}'.format(
cchq_user=cchq_user,
remote_command=shlex_quote(remote_command),
)
]
if args.tee_file:
rc = Ssh(self.parser).run(
args,
_get_ssh_args('cd {code_dir}; [[ -f {tee_file} ]]'.format(
code_dir=code_dir, tee_file=shlex_quote(args.tee_file))))
if rc in (0, 1):
file_already_exists = (rc == 0)
else:
return rc
if file_already_exists:
puts(
color_error(
"Refusing to --tee to a file that already exists ({})".
format(args.tee_file)))
return 1
tee_file_cmd = ' | tee {}'.format(shlex_quote(args.tee_file))
else:
tee_file_cmd = ''
python_env = 'python_env-3.6'
remote_command = (
'cd {code_dir}; {python_env}/bin/python manage.py {args}{tee_file_cmd}'
.format(
python_env=python_env,
cchq_user=cchq_user,
code_dir=code_dir,
args=' '.join(shlex_quote(arg) for arg in manage_args),
tee_file_cmd=tee_file_cmd,
))
if args.tmux:
args.remote_command = remote_command
return Tmux(self.parser).run(args, [])
else:
ssh_args = _get_ssh_args(remote_command)
return Ssh(self.parser).run(args, ssh_args)
def run(self,
app='org.mozilla.geckoview_example',
intent=None,
env=[],
profile=None,
url=None,
no_install=None,
no_wait=None,
fail_if_running=None,
restart=None):
from mozrunner.devices.android_device import verify_android_device, _get_device
from six.moves import shlex_quote
if app == 'org.mozilla.geckoview_example':
activity_name = 'org.mozilla.geckoview_example.GeckoViewActivity'
elif app == 'org.mozilla.geckoview.test':
activity_name = 'org.mozilla.geckoview.test.TestRunnerActivity'
elif 'fennec' in app or 'firefox' in app:
activity_name = 'org.mozilla.gecko.BrowserApp'
else:
raise RuntimeError('Application not recognized: {}'.format(app))
# `verify_android_device` respects `DEVICE_SERIAL` if it is set and sets it otherwise.
verify_android_device(self, app=app, install=not no_install)
device_serial = os.environ.get('DEVICE_SERIAL')
if not device_serial:
print('No ADB devices connected.')
return 1
device = _get_device(self.substs, device_serial=device_serial)
args = []
if profile:
if os.path.isdir(profile):
host_profile = profile
# Always /data/local/tmp, rather than `device.test_root`, because GeckoView only
# takes its configuration file from /data/local/tmp, and we want to follow suit.
target_profile = '/data/local/tmp/{}-profile'.format(app)
device.rm(target_profile, recursive=True, force=True)
device.push(host_profile, target_profile)
self.log(
logging.INFO, "run", {
'host_profile': host_profile,
'target_profile': target_profile
},
'Pushed profile from host "{host_profile}" to target "{target_profile}"'
)
else:
target_profile = profile
self.log(logging.INFO, "run",
{'target_profile': target_profile},
'Using profile from target "{target_profile}"')
args = ['--profile', shlex_quote(target_profile)]
extras = {}
for i, e in enumerate(env):
extras['env{}'.format(i)] = e
if args:
extras['args'] = " ".join(args)
extras[
'use_multiprocess'] = True # Only GVE and TRA process this extra.
if env or args:
restart = True
if restart:
fail_if_running = False
self.log(logging.INFO, "run", {'app': app},
'Stopping {app} to ensure clean restart.')
device.stop_application(app)
# We'd prefer to log the actual `am start ...` command, but it's not trivial to wire the
# device's logger to mach's logger.
self.log(logging.INFO, "run", {
'app': app,
'activity_name': activity_name
}, 'Starting {app}/{activity_name}.')
device.launch_application(app_name=app,
activity_name=activity_name,
intent=intent,
extras=extras,
url=url,
wait=not no_wait,
fail_if_running=fail_if_running)
return 0
def _sanitize_param_dict(param_dict):
return {
str(key): shlex_quote(str(value))
for key, value in param_dict.items()
}
def _str_optional(s):
return "NULL" if s is None else "'{}'".format(shlex_quote(str(s)))
def create_commands(session,
num_workers,
remotes,
env_id,
logdir,
shell='bash',
mode='tmux',
hparams=None):
# for launching the TF workers and for launching tensorboard
base_cmd = [
'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir', logdir, '--env-id',
env_id, '--num-workers', str(num_workers)
]
if hparams is not None:
for k, v in hparams.items():
base_cmd.append("--" + k)
if isinstance(v, list):
base_cmd.extend(v)
elif isinstance(v, str):
base_cmd.append(v)
else:
base_cmd.append(str(v))
os.makedirs(logdir, exist_ok=True)
hparams["env_id"] = env_id
import json
with open(logdir + '/hparams.json', 'w') as f:
json.dump(hparams, f, indent=4)
if remotes is None:
remotes = ["1"] * num_workers
else:
remotes = remotes.split(',')
assert len(remotes) == num_workers
cmds_map = [new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir, shell)]
for i in range(num_workers):
cmds_map += [
new_cmd(session, "w-%d" % i,
base_cmd + ["--job-name", "worker", "--task", str(i), "--remotes", remotes[i]],
mode, logdir, shell)
]
cmds_map += [
new_cmd(session, "tb", ["tensorboard", "--logdir", logdir, "--port", "12345"], mode, logdir,
shell)
]
if mode == 'tmux':
cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)]
windows = [v[0] for v in cmds_map]
notes = []
cmds = [
"mkdir -p {}".format(logdir),
"mkdir -p {}".format(os.path.join(logdir, "logs")),
"echo {} {} > {}/cmd.sh".format(sys.executable, ' '.join(
[shlex_quote(arg) for arg in sys.argv if arg != '-n']), logdir),
]
if mode == 'nohup' or mode == 'child':
cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)]
notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)]
if mode == 'tmux':
notes += ["Use `tmux attach -t {}` to watch process output".format(session)]
notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)]
else:
notes += ["Use `tail -f {}/*.out` to watch process output".format(logdir)]
notes += ["Point your browser to http://localhost:12345 to see Tensorboard"]
if mode == 'tmux':
cmds += [
"tmux kill-session -t {}".format(session), "tmux new-session -s {} -n {} -d {}".format(
session, windows[0], shell)
]
for w in windows[1:]:
cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)]
cmds += ["sleep 1"]
for window, cmd in cmds_map:
cmds += [cmd]
return cmds, notes
def get_compile_command(click_ctx):
"""
Returns a normalized compile command depending on cli context.
The command will be normalized by:
- expanding options short to long
- removing values that are already default
- sorting the arguments
- removing one-off arguments like '--upgrade'
- removing arguments that don't change build behaviour like '--verbose'
"""
from piptools.scripts.compile import cli
# Map of the compile cli options (option name -> click.Option)
compile_options = {option.name: option for option in cli.params}
left_args = []
right_args = []
for option_name, value in click_ctx.params.items():
option = compile_options[option_name]
# Get the latest option name (usually it'll be a long name)
option_long_name = option.opts[-1]
# Collect variadic args separately, they will be added
# at the end of the command later
if option.nargs < 0:
# These will necessarily be src_files
# Re-add click-stripped '--' if any start with '-'
if any(val.startswith("-") and val != "-" for val in value):
right_args.append("--")
right_args.extend([shlex_quote(force_text(val)) for val in value])
continue
# Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
# or options that don't change compile behaviour (--verbose/--dry-run/...)
if option_long_name in COMPILE_EXCLUDE_OPTIONS:
continue
# Skip options without a value
if option.default is None and not value:
continue
# Skip options with a default value
if option.default == value:
continue
# Use a file name for file-like objects
if isinstance(value, LazyFile):
value = value.name
# Convert value to the list
if not isinstance(value, (tuple, list)):
value = [value]
for val in value:
# Flags don't have a value, thus add to args true or false option long name
if option.is_flag:
# If there are false-options, choose an option name depending on a value
if option.secondary_opts:
# Get the latest false-option
secondary_option_long_name = option.secondary_opts[-1]
arg = option_long_name if val else secondary_option_long_name
# There are no false-options, use true-option
else:
arg = option_long_name
left_args.append(shlex_quote(arg))
# Append to args the option with a value
else:
if isinstance(val, six.string_types) and is_url(val):
val = redact_auth_from_url(val)
if option.name == "pip_args":
# shlex_quote would produce functional but noisily quoted results,
# e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
# Instead, we try to get more legible quoting via repr:
left_args.append("{option}={value}".format(
option=option_long_name,
value=repr(fs_str(force_text(val)))))
else:
left_args.append("{option}={value}".format(
option=option_long_name,
value=shlex_quote(force_text(val))))
return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))