def on_click_dispatcher(self, module_name, event, command):
"""
Dispatch on_click config parameters to either:
- Our own methods for special py3status commands (listed below)
- The i3-msg program which is part of i3wm
"""
if command is None:
return
elif command == 'refresh_all':
self.py3_wrapper.refresh_modules()
elif command == 'refresh':
self.py3_wrapper.refresh_modules(module_name)
else:
# In commands we are able to use substitutions for the text output
# of a module
if '$OUTPUT' in command or '$OUTPUT_PART' in command:
full_text, partial_text = self.get_module_text(
module_name, event)
command = command.replace('$OUTPUT_PART',
shell_quote(partial_text))
command = command.replace('$OUTPUT', shell_quote(full_text))
# this is a i3 message
self.i3_msg(module_name, command)
# to make the bar more responsive to users we ask for a refresh
# of the module or of i3status if the module is an i3status one
self.py3_wrapper.refresh_modules(module_name)
def on_click_dispatcher(self, module_name, event, command):
"""
Dispatch on_click config parameters to either:
- Our own methods for special py3status commands (listed below)
- The i3-msg program which is part of i3wm
"""
if command is None:
return
elif command == 'refresh_all':
self.py3_wrapper.refresh_modules()
elif command == 'refresh':
self.py3_wrapper.refresh_modules(module_name)
else:
# In commands we are able to use substitutions for the text output
# of a module
if '$OUTPUT' in command or '$OUTPUT_PART' in command:
full_text, partial_text = self.get_module_text(module_name,
event)
command = command.replace('$OUTPUT_PART',
shell_quote(partial_text))
command = command.replace('$OUTPUT', shell_quote(full_text))
# this is a i3 message
self.i3_msg(module_name, command)
# to make the bar more responsive to users we ask for a refresh
# of the module or of i3status if the module is an i3status one
self.py3_wrapper.refresh_modules(module_name)
def upload(self, release_id):
from fabric.contrib import files
import posixpath
upload_storage_path = self._abs_path(self.upload_storage_path)
self._ensure_path_exists(upload_storage_path)
upload_release_path = self._get_upload_release_path(release_id)
self._ensure_path_exists(upload_release_path)
upload_storage_current_link = self._path_join(upload_storage_path, self.upload_storage_current_link)
for local_path, remote_path in self.upload_paths:
if files.exists(remote_path) and not files.is_link(remote_path):
raise RuntimeError('Remote path already exists, but is no symlink (%s)' % remote_path)
remote_upload_path = self._get_remote_upload_path(release_id, remote_path)
if self._exists(upload_storage_current_link) and not self._exists(remote_upload_path):
remote_upload_pathname = self._get_remote_upload_pathname(release_id, remote_path)
active_remote_upload_path = self._path_join(upload_storage_current_link, remote_upload_pathname)
cp_active_remote_upload_path = active_remote_upload_path.rstrip(posixpath.sep) + posixpath.sep
cp_remote_upload_path = remote_upload_path.rstrip(posixpath.sep) + posixpath.sep
self._run('{cp_bin} {cp_args} {from_path} {to_path}'.format(
cp_bin=self.cp_bin(),
cp_args=self.cp_args,
from_path=shell_quote(cp_active_remote_upload_path),
to_path=shell_quote(cp_remote_upload_path),
))
self._rsync_upload(local_path, remote_upload_path)
def run_test_cmd(self, cmd, log):
cmd = ". bin/activate {0}; {1}".format(shell_quote(self.env_dir), cmd)
cmd = "bash -c {0}".format(shell_quote(cmd))
self.print("$ cd cache/env; " + cmd, level=1)
subprocess.call(cmd, stdout=log, stderr=subprocess.STDOUT, shell=True,
cwd=self.env_dir)
def run_test_cmd(self, cmd, log):
cmd = ". bin/activate {0}; {1}".format(shell_quote(self.env_dir), cmd)
cmd = "bash -c {0}".format(shell_quote(cmd))
self.print("$ cd cache/env; " + cmd, level=1)
subprocess.call(cmd, stdout=log, stderr=log, shell=True,
cwd=self.env_dir)
def handle(self, event, container_id):
cmd="%s %s %s %s" % (
self.resolved_script,
shell_quote(self._ini_file),
shell_quote(event),
shell_quote(container_id),
)
logger.debug('running %r', cmd)
status=os.system(cmd)
logger.debug('script of %r returned %r', self._base_name, status)
def convert_epub_to_html(input_path_storage=None, logger=None):
"""Convert epub to html.
Parameters
----------
input_path_storage : None, optional
Path to where .epub files are stored.
logger : LogSystem
The logger.
"""
input_path = file_utils.expand_path(input_path_storage) if input_path_storage else \
os.path.join(PATHS["convertions"], "epub_to_html")
for dirname, dirnames, filenames in os.walk(input_path, topdown=False):
for filename in filenames:
if filename.endswith(".epub"):
f_path = os.path.join(dirname, filename)
f_name = os.path.basename(f_path)
dst_name = os.path.splitext(f_name)[0]
dst_path = os.path.join(os.path.dirname(f_path), dst_name)
dst_file = os.path.join(dst_path, "index.html")
if file_utils.is_real_file(dst_file):
continue
logger.info(shell_utils.get_cli_separator("-"), date=False)
logger.info("**Converting:**")
logger.info(f_path, date=False)
os.makedirs(dst_path, mode=0o777, exist_ok=True)
cmd = [
"pandoc", "--standalone",
shell_quote(f_path), "--output",
shell_quote(dst_file),
"--css=/assets/css/bootstrap.min.css",
"--css=/assets/css/bootstrap.tweaks.css",
"--extract-media=assets",
"--template=%s" %
shell_quote(PATHS["pandoc_html_template"]), "--wrap=none",
"--no-highlight", "--table-of-contents", "--to=html5"
]
try:
logger.info(" ".join(cmd), date=False)
cmd_utils.run_cmd(" ".join(cmd),
stdout=None,
stderr=None,
cwd=dst_path,
shell=True,
check=True)
except CalledProcessError as err:
logger.error(err)
def run(self,
args,
stdin_string=None,
env_extend=None,
binary_output=False):
# Allow overriding default settings. If a piece of code really wants to
# set own PATH or CIB_file, we must allow it. I.e. it wants to run
# a pacemaker tool on a CIB in a file but cannot afford the risk of
# changing the CIB in the file specified by the user.
env_vars = self._env_vars.copy()
env_vars.update(dict(env_extend) if env_extend else dict())
log_args = " ".join([shell_quote(x) for x in args])
self._logger.debug(
"Running: {args}\nEnvironment:{env_vars}{stdin_string}".format(
args=log_args,
stdin_string=("" if not stdin_string else (
"\n--Debug Input Start--\n{0}\n--Debug Input End--".format(
stdin_string))),
env_vars=("" if not env_vars else ("\n" + "\n".join([
" {0}={1}".format(key, val)
for key, val in sorted(env_vars.items())
])))))
self._reporter.process(
reports.run_external_process_started(log_args, stdin_string,
env_vars))
try:
process = subprocess.Popen(
args,
# Some commands react differently if they get anything via stdin
stdin=(subprocess.PIPE
if stdin_string is not None else subprocess.DEVNULL),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=(
lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL)),
close_fds=True,
shell=False,
env=env_vars,
# decodes newlines and in python3 also converts bytes to str
universal_newlines=(not binary_output))
out_std, out_err = process.communicate(stdin_string)
retval = process.returncode
except OSError as e:
raise LibraryError(
reports.run_external_process_error(log_args, e.strerror))
self._logger.debug(
("Finished running: {args}\nReturn value: {retval}" +
"\n--Debug Stdout Start--\n{out_std}\n--Debug Stdout End--" +
"\n--Debug Stderr Start--\n{out_err}\n--Debug Stderr End--"
).format(args=log_args,
retval=retval,
out_std=out_std,
out_err=out_err))
self._reporter.process(
reports.run_external_process_finished(log_args, retval, out_std,
out_err))
return out_std, out_err, retval
def as_shell(self):
cmd = ' '.join(shell_quote(i) for i in self.args)
if self.pre_pipe:
cmd = '{} | {}'.format(self.pre_pipe, cmd)
if self.post_pipe:
cmd = '{} | {}'.format(cmd, self.post_pipe)
return cmd
def gen_command_line_args(self, parser_arguments):
"""Generate a gen.py argument list to be embedded in a Ninja file."""
result = []
for args, kwargs in self._arguments:
if len(args) == 2:
long_option = args[1]
else:
long_option = args[0]
dest = kwargs.get('dest', None)
if dest is None:
assert long_option.startswith('--')
dest = long_option[2:].replace('-', '_')
if getattr(parser_arguments, dest, None) is None:
# This was not set on the command-line so skip it.
continue
action = kwargs.get('action', None)
if action == 'store_true':
if getattr(parser_arguments, dest):
result.append(long_option)
elif action == 'store' or action is None:
result.append('%s=%s' %
(long_option, getattr(parser_arguments, dest)))
elif action == 'append':
for item in getattr(parser_arguments, dest):
result.append('%s=%s' % (long_option, item))
else:
assert action is None, "Unsupported action " + action
return ' '.join(shell_quote(item) for item in result)
def switch_release(self, release_id):
git_bin = self.git_bin()
for local_path, remote_path in self.upload_paths:
with self._cd(remote_path):
self._run('{git_bin} checkout {sha}'.format(
git_bin=git_bin,
sha=shell_quote(self._release_tag(release_id)),
))
def switch_release(self, release_id):
git_bin = self.git_bin()
for local_path, remote_path in self.upload_paths:
with self._cd(remote_path):
self._run('{git_bin} checkout {sha}'.format(
git_bin=git_bin,
sha=shell_quote(self._release_tag(release_id)),
))
def value_str(s): if isinstance(s, bool): return 'true' if s else 'false' if isinstance(s, int): return str(s) if not s: return s return shell_quote(str(s))
def upload(self, release_id, commit_message=None):
import datetime
warnings.warn(
'The GitUploader has no way to follow the normal upload schema where changes to the files are '
'only done when calling switch_release. This means the upload already applies the changes and '
'then reverts back to the original state. THERE MIGHT BE DRAGONS!')
if commit_message is None:
commit_message = datetime.datetime.now().isoformat()
# TODO: Make sure no changes are done to the file system while uploading, which is bad design ;-)
for local_path, remote_path in self.upload_paths:
self._ensure_path_exists(remote_path)
git_bin = self.git_bin()
with self._cd(remote_path):
current_release_sha = None
if not self._exists(self._path_join(remote_path, '.git')):
self._run('{git_bin} init .'.format(git_bin=git_bin))
else:
current_release_sha = self._run(
'{git_bin} rev-parse HEAD'.format(git_bin=git_bin))
self._run('{git_bin} checkout master'.format(git_bin=git_bin))
self._rsync_upload(local_path, remote_path, exclude=('.git', ))
self._run('{git_bin} add -A'.format(git_bin=git_bin))
self._run(
'{git_bin} ls-files --deleted -z | xargs -r -0 git rm'.
format(git_bin=git_bin))
with fab.settings(warn_only=True):
self._run(
'{git_bin} commit --allow-empty -m {commit_message}'.
format(
git_bin=git_bin,
commit_message=shell_quote(commit_message),
))
self._run('{git_bin} tag {release_id}'.format(
git_bin=git_bin,
release_id=shell_quote(self._release_tag(release_id)),
))
if current_release_sha: # rollback to previous release
self._run('{git_bin} checkout {sha}'.format(
git_bin=git_bin,
sha=shell_quote(current_release_sha),
))
def get_compilation_commands(self,
source_filenames, executable_filename,
for_evaluation=True):
"""See Language.get_compilation_commands."""
compile_command = ["/usr/bin/javac"] + source_filenames
# We need to let the shell expand *.class as javac create
# a class file for each inner class.
if JavaJDK.USE_JAR:
jar_command = ["/bin/sh", "-c",
" ".join(["jar", "cf",
shell_quote(executable_filename),
"*.class"])]
return [compile_command, jar_command]
else:
zip_command = ["/bin/sh", "-c",
" ".join(["zip", "-r", "-", "*.class", ">",
shell_quote(executable_filename)])]
return [compile_command, zip_command]
def parse_cflags(raw_cflags):
raw_args = shlex.split(raw_cflags.strip())
args, unknown = _cflag_parser.parse_known_args(raw_args)
config = {k: v or [] for k, v in args.__dict__.items()}
for i, x in enumerate(config['define_macros']):
parts = x.split('=', 1)
value = x[1] or None if len(x) == 2 else None
config['define_macros'][i] = (parts[0], value)
return config, ' '.join(shell_quote(x) for x in unknown)
def remote_notify(self, msg):
"""Display a message remotely."""
try:
notify_cmd = vim.vars['skuld_notify_cmd']
except KeyError:
notify_cmd = None
if notify_cmd is not None and len(notify_cmd) > 0:
os.system(notify_cmd + ' ' + shell_quote(msg))
else:
remote_cmd = ("<c-\\><c-n>"
+ ":echohl WarningMsg | echo ''{0}''"
+ " | echohl None "
+ " | call foreground() "
+ " | SkuldBufUpdate<cr>").format(msg)
vim_cmd = "call remote_send('{0}', '{1}')".format(
self._vim_server_name, remote_cmd)
sys_cmd = "gvim --cmd {0} --cmd qa".format(shell_quote(vim_cmd))
os.system(sys_cmd)
def parse_cflags(raw_cflags):
raw_args = shlex.split(raw_cflags.strip())
args, unknown = _cflag_parser.parse_known_args(raw_args)
config = {k: v or [] for k, v in args.__dict__.items()}
for i, x in enumerate(config['define_macros']):
parts = x.split('=', 1)
value = x[1] or None if len(x) == 2 else None
config['define_macros'][i] = (parts[0], value)
return config, ' '.join(shell_quote(x) for x in unknown)
def get_compilation_commands(self,
source_filenames, executable_filename,
for_evaluation=True):
"""See Language.get_compilation_commands."""
compile_command = ["/usr/bin/javac"] + source_filenames
# We need to let the shell expand *.class as javac create
# a class file for each inner class.
if JavaJDK.USE_JAR:
jar_command = ["/bin/sh", "-c",
" ".join(["jar", "cf",
shell_quote(executable_filename),
"*.class"])]
return [compile_command, jar_command]
else:
zip_command = ["/bin/sh", "-c",
" ".join(["zip", "-r", "-", "*.class", ">",
shell_quote(executable_filename)])]
return [compile_command, zip_command]
def _test_fuzz(workdir, target, dumb=False):
require_commands('py-afl-fuzz', 'afl-fuzz')
input_dir = workdir + '/in'
output_dir = workdir + '/out'
os.mkdir(input_dir)
os.mkdir(output_dir)
with open(input_dir + '/in', 'w') as file:
file.write('0')
crash_dir = output_dir + '/crashes'
queue_dir = output_dir + '/queue'
have_crash = False
have_paths = False
n_paths = 0
with open('/dev/null', 'wb') as devnull:
with open(workdir + '/stdout', 'wb') as stdout:
cmdline = [
'py-afl-fuzz', '-i', input_dir, '-o', output_dir, '--',
sys.executable, target, token
]
if dumb:
cmdline[1:1] = ['-n']
print('$ ' + ' '.join(shell_quote(arg) for arg in cmdline))
afl = ipc.Popen(
cmdline,
stdout=stdout,
stdin=devnull,
preexec_fn=clean_environ,
)
try:
timeout = 10
while timeout > 0:
if afl.poll() is not None:
break
have_crash = len(glob.glob(crash_dir + '/id:*')) >= 1
n_paths = len(glob.glob(queue_dir + '/id:*'))
have_paths = (n_paths == 1) if dumb else (n_paths >= 2)
if have_crash and have_paths:
break
timeout -= sleep(0.1)
if afl.returncode is None:
afl.terminate()
afl.wait()
except:
afl.kill()
raise
with open(workdir + '/stdout', 'rb') as file:
stdout = file.read()
if str is not bytes:
stdout = stdout.decode('ASCII', 'replace')
print(stdout)
if not have_crash and '/proc/sys/kernel/core_pattern' in stdout:
check_core_pattern()
assert_true(have_crash, "target program didn't crash")
assert_true(have_paths,
'target program produced {n} distinct paths'.format(n=n_paths))
async def _shell_job(self, event, task_id):
maskname = event.maskname.split("|", 1)[0]
if hasattr(event, "src_pathname"):
src_pathname = event.src_pathname
else:
src_pathname = ""
cmd = self._cmd.replace("{maskname}", shell_quote(maskname)).replace(
"{pathname}",
shell_quote(event.pathname)).replace("{src_pathname}",
shell_quote(src_pathname))
logger = SchedulerLogger(self._log, {"event": event, "id": task_id})
logger.info(f"execute shell command, cmd={cmd}")
try:
proc = await asyncio.create_subprocess_shell(cmd, loop=self._loop)
await proc.communicate()
except Exception as e:
logger.error(e)
def upload(self, release_id, commit_message=None):
import datetime
warnings.warn(
'The GitUploader has no way to follow the normal upload schema where changes to the files are '
'only done when calling switch_release. This means the upload already applies the changes and '
'then reverts back to the original state. THERE MIGHT BE DRAGONS!'
)
if commit_message is None:
commit_message = datetime.datetime.now().isoformat()
# TODO: Make sure no changes are done to the file system while uploading, which is bad design ;-)
for local_path, remote_path in self.upload_paths:
self._ensure_path_exists(remote_path)
git_bin = self.git_bin()
with self._cd(remote_path):
current_release_sha = None
if not self._exists(self._path_join(remote_path, '.git')):
self._run('{git_bin} init .'.format(git_bin=git_bin))
else:
current_release_sha = self._run('{git_bin} rev-parse HEAD'.format(git_bin=git_bin))
self._run('{git_bin} checkout master'.format(git_bin=git_bin))
self._rsync_upload(local_path, remote_path, exclude=('.git',))
self._run('{git_bin} add -A'.format(git_bin=git_bin))
self._run('{git_bin} ls-files --deleted -z | xargs -r -0 git rm'.format(git_bin=git_bin))
with fab.settings(warn_only=True):
self._run('{git_bin} commit --allow-empty -m {commit_message}'.format(
git_bin=git_bin,
commit_message=shell_quote(commit_message),
))
self._run('{git_bin} tag {release_id}'.format(
git_bin=git_bin,
release_id=shell_quote(self._release_tag(release_id)),
))
if current_release_sha: # rollback to previous release
self._run('{git_bin} checkout {sha}'.format(
git_bin=git_bin,
sha=shell_quote(current_release_sha),
))
def read_shell_config(config_path=None, whitelisted_keys=[]):
commands = []
if config_path is not None:
commands.append(". %s" % shell_quote(config_path))
commands.append("cat /proc/self/environ")
for key in whitelisted_keys:
commands.append(
"test -z \"${KEY+x}\" || printf '%s\\0' \"KEY=$KEY\"".replace(
"KEY", key))
env = _read_nullsep_output(["sh", "-c", "; ".join(commands)])
for key, value in env.items():
yield key, value
def switch_release(self, release_id):
upload_storage_path = self._abs_path(self.upload_storage_path)
upload_storage_current_link = self._path_join(upload_storage_path, self.upload_storage_current_link)
# make sure all symlinks are working
for local_path, remote_path in self.upload_paths:
remote_upload_pathname = self._get_remote_upload_pathname(release_id, remote_path)
if self._exists(remote_path):
self._run('{rm_bin} {path}'.format(
rm_bin=self.rm_bin(),
path=shell_quote(remote_path),
))
self._run('{ln_bin} {ln_args} {current_link} {upload_path}'.format(
ln_bin=self.ln_bin(),
ln_args=self.ln_args,
current_link=shell_quote(self._path_join(upload_storage_current_link, remote_upload_pathname)),
upload_path=shell_quote(remote_path),
))
# switch to current release
if self._exists(upload_storage_current_link):
self._run('{rm_bin} {path}'.format(
rm_bin=self.rm_bin(),
path=shell_quote(upload_storage_current_link),
))
self._run('{ln_bin} {ln_args} {release_path} {current_link}'.format(
ln_bin=self.ln_bin(),
ln_args=self.ln_args,
release_path=shell_quote(self._get_upload_release_pathname(release_id)),
current_link=shell_quote(upload_storage_current_link),
))
def zfs_command(self, *args, pre_pipe=None, post_pipe=None):
assert args[0] == 'zfs', 'Only supported zfs arg for now'
remote_args = ('sudo', 'zfs') + tuple(args[1:])
# Make the call into a single argument: "'zfs' 'list' '...'"
remote_arg = ' '.join(shell_quote(i) for i in remote_args)
# Put pre_pipe and post_pipe inside the (remote) shell command.
if pre_pipe:
remote_arg = '{} | {}'.format(pre_pipe, remote_arg)
if post_pipe:
remote_arg = '{} | {}'.format(remote_arg, post_pipe)
# Prepend remote args: ('ssh', 'user@host', "'zfs' 'list' '...'")
args = self._run_remote_args + (remote_arg,)
return ZfsCommand(args, pre_pipe=None, post_pipe=None)
def upload(self, release_id):
from fabric.contrib import files
import posixpath
upload_storage_path = self._abs_path(self.upload_storage_path)
self._ensure_path_exists(upload_storage_path)
upload_release_path = self._get_upload_release_path(release_id)
self._ensure_path_exists(upload_release_path)
upload_storage_current_link = self._path_join(
upload_storage_path, self.upload_storage_current_link)
for local_path, remote_path in self.upload_paths:
if files.exists(remote_path) and not files.is_link(remote_path):
raise RuntimeError(
'Remote path already exists, but is no symlink (%s)' %
remote_path)
remote_upload_path = self._get_remote_upload_path(
release_id, remote_path)
if self._exists(upload_storage_current_link
) and not self._exists(remote_upload_path):
remote_upload_pathname = self._get_remote_upload_pathname(
release_id, remote_path)
active_remote_upload_path = self._path_join(
upload_storage_current_link, remote_upload_pathname)
cp_active_remote_upload_path = active_remote_upload_path.rstrip(
posixpath.sep) + posixpath.sep
cp_remote_upload_path = remote_upload_path.rstrip(
posixpath.sep) + posixpath.sep
self._run('{cp_bin} {cp_args} {from_path} {to_path}'.format(
cp_bin=self.cp_bin(),
cp_args=self.cp_args,
from_path=shell_quote(cp_active_remote_upload_path),
to_path=shell_quote(cp_remote_upload_path),
))
self._rsync_upload(local_path, remote_upload_path)
def _test_fuzz(workdir, target, dumb=False):
require_commands('py-afl-fuzz', 'afl-fuzz')
input_dir = workdir + '/in'
output_dir = workdir + '/out'
os.mkdir(input_dir)
os.mkdir(output_dir)
with open(input_dir + '/in', 'w') as file:
file.write('0')
crash_dir = output_dir + '/crashes'
queue_dir = output_dir + '/queue'
have_crash = False
have_paths = False
n_paths = 0
with open('/dev/null', 'wb') as devnull:
with open(workdir + '/stdout', 'wb') as stdout:
cmdline = ['py-afl-fuzz', '-i', input_dir, '-o', output_dir, '--', sys.executable, target, token]
if dumb:
cmdline[1:1] = ['-n']
print('$ ' + ' '.join(shell_quote(arg) for arg in cmdline))
afl = ipc.Popen(
cmdline,
stdout=stdout,
stdin=devnull,
preexec_fn=clean_environ,
)
try:
timeout = 10
while timeout > 0:
if afl.poll() is not None:
break
have_crash = len(glob.glob(crash_dir + '/id:*')) >= 1
n_paths = len(glob.glob(queue_dir + '/id:*'))
have_paths = (n_paths == 1) if dumb else (n_paths >= 2)
if have_crash and have_paths:
break
timeout -= sleep(0.1)
if afl.returncode is None:
afl.terminate()
afl.wait()
except:
afl.kill()
raise
with open(workdir + '/stdout', 'rb') as file:
stdout = file.read()
if str is not bytes:
stdout = stdout.decode('ASCII', 'replace')
print(stdout)
if not have_crash and '/proc/sys/kernel/core_pattern' in stdout:
check_core_pattern()
assert_true(have_crash, "target program didn't crash")
assert_true(have_paths, 'target program produced {n} distinct paths'.format(n=n_paths))
def test_repo_accessible_through_proxy(self):
PORT = '4000'
proxy = shell_quote(self.PROXY_PATH)
key = shell_quote(self.SSH_KEY)
self.env['PROXY_CMD'] = '{} {} -i {} --no-host-key-check relay %p %h %r'.format(PYTHON, proxy, key)
self.env['GIT_SSH'] = os.path.join(self.ROOT_DIR, 'tests', 'git_ssh_proxy.sh')
self.env['PYTHONPATH'] = self.ROOT_DIR
self.env['PORT'] = PORT
server = None
try:
# run the server
server_cmd = os.path.join(self.ROOT_DIR, 'bin', 'simple-ssh-server.py')
server = subprocess.Popen([PYTHON, server_cmd, PORT], env=self.env)
# wait a second
time.sleep(1)
self.assertIsNone( server.poll() )
self.check_git_clone(self.env)
finally:
if server:
server.kill()
def argsjoin(cmd):
"""
Return cmd-tuple as a quoted string, safe to pass to a shell.
"""
def is_safe(arg):
return all(i in ('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
'09123456789_=-+.,/~') for i in arg)
args = []
for arg in cmd:
if is_safe(arg):
args.append(arg)
else:
args.append(shell_quote(arg))
return ' '.join(args)
def shell(self, cmd, *args):
"""Call to os.system in 'safe mode'
Replaced by subprocess.call so we can ensure use of BASH. Using DASH breaks everything.
"""
full_cmd = "set -euo pipefail ; "
if args:
full_cmd += cmd.format(*[shell_quote(a) for a in args])
else:
# Assume that any curlies are bash expansions
full_cmd += cmd
status = subprocess.call(full_cmd, shell=True, executable="/bin/bash")
if status:
raise ChildProcessError("Exit status was %s running command:\n%s" %
(status, cmd))
return status
def main() -> None:
"""Main entry point for the simple compiler test harness."""
if 'SC' in environ:
cmd = "{} --sc {}".format(argv[0], shell_quote(environ['SC']))
warn("The SC environment variable is deprecated. Use: {}".format(cmd),
DeprecationWarning)
argv[1:1] = ['--sc', environ['SC']]
args = _get_args()
test_runner = TextTestRunner(verbosity=args.verbosity)
test_suite = TestSuite([p(**args.__dict__) for p in args.phases])
test_runner.run(test_suite)
def init_docker_host(context, docker):
# install ovs ; do in base image Dockerfile
# start ovs server
cmd = "/usr/local/share/openvswitch/scripts/ovs-ctl start --system-id=random"
call_in_docker(docker, cmd)
# copy wheel file
vlcp_wheel = "*.whl"
if "vlcp" in context.config.userdata:
vlcp_wheel = context.config.userdata["vlcp"]
cmd = "docker cp %s %s:/opt" % (vlcp_wheel, docker)
subprocess.check_call(cmd, shell=True)
# install vlcp
cmd = "/opt/pip install --upgrade /opt/%s" % vlcp_wheel
c = "docker exec %s bash -c %s" % (docker, shell_quote(cmd))
subprocess.check_output(c, shell=True)
if "coverage" in context.config.userdata:
cmd = "docker cp %s %s:/opt" % ("coverage.conf", docker)
subprocess.check_call(cmd, shell=True)
cmd = "sed -i 's~/opt/python~/opt/coverage run --rcfile=/opt/coverage.conf~g' %s" % "supervisord.conf"
subprocess.check_call(cmd, shell=True)
else:
cmd = "sed -i 's~/opt/coverage run --rcfile=/opt/coverage.conf~/opt/python~g' %s" % "supervisord.conf"
subprocess.check_call(cmd, shell=True)
# copy supervisor conf to host
cmd = "docker cp %s %s:/etc" % ("supervisord.conf",docker)
subprocess.check_call(cmd, shell=True)
# start supervisord
cmd = "supervisord -c /etc/supervisord.conf"
call_in_docker(docker, cmd)
# add ovs bridge br0
cmd = "ovs-vsctl add-br br0"
call_in_docker(docker, cmd)
# set br0 controller to 127.0.0.1
cmd = "ovs-vsctl set-controller br0 tcp:127.0.0.1"
call_in_docker(docker, cmd)
def lines():
line = ''
for a in (shell_quote(a) for a in args):
# If adding this argument will make the line too long,
# yield the current line, and start a new one.
if len(line) + len(a) + 1 > maxwidth:
yield line
line = ''
# Append this argument to the current line, separating
# it by a space from the existing arguments.
if line:
line += ' ' + a
else:
line = a
yield line
def lines():
line = ''
for a in (shell_quote(a) for a in args):
# If adding this argument will make the line too long,
# yield the current line, and start a new one.
if len(line) + len(a) + 1 > maxwidth:
yield line
line = ''
# Append this argument to the current line, separating
# it by a space from the existing arguments.
if line:
line += ' ' + a
else:
line = a
yield line
def execute_subprocess(args, log=None, failure=None, **kwargs):
"""Run `args`, handling logging and failures.
:param args: Program argument array.
:param log: An optional logger.
:param failure: Raise `failure` as an exception if the command returns a
nonzero value. If omitted, nonzero return values are ignored.
:param kwargs: Other keyword arguments passed on to `subprocess.call`.
"""
if log is not None:
log.debug("Executing: %s", " ".join(shell_quote(arg) for arg in args))
retval = subprocess.call(args, **kwargs)
if retval != 0:
if log is not None:
log.debug("Command returned %d.", retval)
if failure is not None:
if log is not None:
log.debug("Command failed: %s", failure)
raise failure
def shell(prog, *args, **kwargs):
"""Runs a shell command.
Positional arguments are used as command arguments.
Keyword arguments are used as flags.
:return: Output of the command (stdout)
"""
cmd_parts = [prog]
cmd_parts.extend(map(shell_quote, args))
for flag, value in kwargs.iteritems():
prefix = '-' if len(flag) == 1 else '--'
cmd_parts.append((prefix + flag.replace('_', '-')))
if value and value is not True:
cmd_parts.append(shell_quote(value))
cmd = ' '.join(cmd_parts)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
out, _ = process.communicate()
return out
def prettyformat_shell_command(self, executable, args=None, kwargs=None, _cwd=None):
output = [executable]
if args:
output.extend([shell_quote(arg) for arg in args])
if kwargs:
for key, value in kwargs.items():
if value:
if len(key) == 1:
prefix = '-'
else:
prefix = '--'
if value is True:
formatted_value = ''
else:
formatted_value = '={}'.format(value)
output.append('{prefix}{key}{formatted_value}'.format(
prefix=prefix,
key=key,
formatted_value=formatted_value
))
if _cwd:
output.insert(0, '[CWD={}]'.format(_cwd))
return ' '.join(output)
def switch_release(self, release_id):
upload_storage_path = self._abs_path(self.upload_storage_path)
upload_storage_current_link = self._path_join(
upload_storage_path, self.upload_storage_current_link)
# make sure all symlinks are working
for local_path, remote_path in self.upload_paths:
remote_upload_pathname = self._get_remote_upload_pathname(
release_id, remote_path)
if self._exists(remote_path):
self._run('{rm_bin} {path}'.format(
rm_bin=self.rm_bin(),
path=shell_quote(remote_path),
))
self._run('{ln_bin} {ln_args} {current_link} {upload_path}'.format(
ln_bin=self.ln_bin(),
ln_args=self.ln_args,
current_link=shell_quote(
self._path_join(upload_storage_current_link,
remote_upload_pathname)),
upload_path=shell_quote(remote_path),
))
# switch to current release
if self._exists(upload_storage_current_link):
self._run('{rm_bin} {path}'.format(
rm_bin=self.rm_bin(),
path=shell_quote(upload_storage_current_link),
))
self._run('{ln_bin} {ln_args} {release_path} {current_link}'.format(
ln_bin=self.ln_bin(),
ln_args=self.ln_args,
release_path=shell_quote(
self._get_upload_release_pathname(release_id)),
current_link=shell_quote(upload_storage_current_link),
))
def WriteGenericNinja(path,
static_libraries,
executables,
cxx,
ar,
ld,
platform,
host,
options,
cflags=[],
ldflags=[],
libflags=[],
include_dirs=[],
solibs=[]):
args = ' -d' if options.debug else ''
for link_lib in options.link_libs:
args += ' --link-lib=' + shell_quote(link_lib)
ninja_header_lines = [
'cxx = ' + cxx,
'ar = ' + ar,
'ld = ' + ld,
'',
'rule regen',
' command = %s ../build/gen.py%s' % (sys.executable, args),
' description = Regenerating ninja files',
'',
'build build.ninja: regen',
' generator = 1',
' depfile = build.ninja.d',
'',
]
template_filename = os.path.join(
SCRIPT_DIR, {
'msvc': 'build_win.ninja.template',
'mingw': 'build_linux.ninja.template',
'msys': 'build_linux.ninja.template',
'darwin': 'build_mac.ninja.template',
'linux': 'build_linux.ninja.template',
'freebsd': 'build_linux.ninja.template',
'aix': 'build_aix.ninja.template',
'openbsd': 'build_openbsd.ninja.template',
'haiku': 'build_haiku.ninja.template',
'solaris': 'build_linux.ninja.template',
'netbsd': 'build_linux.ninja.template',
}[platform.platform()])
with open(template_filename) as f:
ninja_template = f.read()
if platform.is_windows():
executable_ext = '.exe'
library_ext = '.lib'
object_ext = '.obj'
else:
executable_ext = ''
library_ext = '.a'
object_ext = '.o'
def escape_path_ninja(path):
return path.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
def src_to_obj(path):
return escape_path_ninja('%s' % os.path.splitext(path)[0] + object_ext)
def library_to_a(library):
return '%s%s' % (library, library_ext)
ninja_lines = []
def build_source(src_file, settings):
ninja_lines.extend([
'build %s: cxx %s' %
(src_to_obj(src_file),
escape_path_ninja(
os.path.relpath(os.path.join(REPO_ROOT, src_file),
os.path.dirname(path)))),
' includes = %s' % ' '.join([
'-I' + escape_path_ninja(dirname) for dirname in include_dirs
]),
' cflags = %s' % ' '.join(cflags),
])
for library, settings in static_libraries.items():
for src_file in settings['sources']:
build_source(src_file, settings)
ninja_lines.append(
'build %s: alink_thin %s' % (library_to_a(library), ' '.join(
[src_to_obj(src_file) for src_file in settings['sources']])))
ninja_lines.append(' libflags = %s' % ' '.join(libflags))
for executable, settings in executables.items():
for src_file in settings['sources']:
build_source(src_file, settings)
ninja_lines.extend([
'build %s%s: link %s | %s' % (executable, executable_ext, ' '.join(
[src_to_obj(src_file)
for src_file in settings['sources']]), ' '.join(
[library_to_a(library) for library in settings['libs']])),
' ldflags = %s' % ' '.join(ldflags),
' solibs = %s' % ' '.join(solibs),
' libs = %s' %
' '.join([library_to_a(library) for library in settings['libs']]),
])
ninja_lines.append('') # Make sure the file ends with a newline.
with open(path, 'w') as f:
f.write('\n'.join(ninja_header_lines))
f.write(ninja_template)
f.write('\n'.join(ninja_lines))
with open(path + '.d', 'w') as f:
f.write('build.ninja: ' + os.path.relpath(
os.path.join(SCRIPT_DIR, 'gen.py'), os.path.dirname(path)) + ' ' +
os.path.relpath(template_filename, os.path.dirname(path)) +
'\n')
def input(self, **kwargs):
encoding = self.default_encoding
options = dict(self.options)
if self.infile is None and "{infile}" in self.command:
# create temporary input file if needed
if self.filename is None:
self.infile = NamedTemporaryFile(mode='wb')
self.infile.write(self.content.encode(encoding))
self.infile.flush()
options["infile"] = self.infile.name
else:
# we use source file directly, which may be encoded using
# something different than utf8. If that's the case file will
# be included with charset="something" html attribute and
# charset will be available as filter's charset attribute
encoding = self.charset # or self.default_encoding
self.infile = open(self.filename)
options["infile"] = self.filename
if "{outfile}" in self.command and "outfile" not in options:
# create temporary output file if needed
ext = self.type and ".%s" % self.type or ""
self.outfile = NamedTemporaryFile(mode='r+', suffix=ext)
options["outfile"] = self.outfile.name
# Quote infile and outfile for spaces etc.
if "infile" in options:
options["infile"] = shell_quote(options["infile"])
if "outfile" in options:
options["outfile"] = shell_quote(options["outfile"])
try:
command = self.command.format(**options)
proc = subprocess.Popen(
command, shell=True, cwd=self.cwd, stdout=self.stdout,
stdin=self.stdin, stderr=self.stderr)
if self.infile is None:
# if infile is None then send content to process' stdin
filtered, err = proc.communicate(
self.content.encode(encoding))
else:
filtered, err = proc.communicate()
filtered, err = filtered.decode(encoding), err.decode(encoding)
except (IOError, OSError) as e:
raise FilterError('Unable to apply %s (%r): %s' %
(self.__class__.__name__, self.command, e))
else:
if proc.wait() != 0:
# command failed, raise FilterError exception
if not err:
err = ('Unable to apply %s (%s)' %
(self.__class__.__name__, self.command))
if filtered:
err += '\n%s' % filtered
raise FilterError(err)
if self.verbose:
self.logger.debug(err)
outfile_path = options.get('outfile')
if outfile_path:
with io.open(outfile_path, 'r', encoding=encoding) as file:
filtered = file.read()
finally:
if self.infile is not None:
self.infile.close()
if self.outfile is not None:
self.outfile.close()
return smart_text(filtered)
def cmd_systemd(args):
conf = load_conf(args)
template = textwrap.dedent("""\
[Unit]
Description=Fishnet instance
After=network.target
[Service]
User={user}
Group={group}
WorkingDirectory={cwd}
Environment=PATH={path}
ExecStart={start}
KillSignal=SIGINT
Restart=always
[Install]
WantedBy=multi-user.target""")
config_file = os.path.abspath(args.conf or DEFAULT_CONFIG)
# Prepare command line arguments
builder = [shell_quote(sys.executable), shell_quote(os.path.abspath(sys.argv[0]))]
if not args.no_conf:
builder.append("--conf")
builder.append(shell_quote(os.path.abspath(args.conf or DEFAULT_CONFIG)))
else:
builder.append("--no-conf")
if args.key is not None:
builder.append("--key")
builder.append(shell_quote(validate_key(args.key, conf)))
if args.engine_dir is not None:
builder.append("--engine-dir")
builder.append(shell_quote(validate_engine_dir(args.engine_dir)))
if args.engine_command is not None:
builder.append("--engine-command")
builder.append(shell_quote(validate_engine_command(args.engine_command, conf)))
if args.cores is not None:
builder.append("--cores")
builder.append(shell_quote(str(validate_cores(args.cores))))
if args.memory is not None:
builder.append("--memory")
builder.append(shell_quote(str(validate_memory(args.memory, conf))))
if args.threads is not None:
builder.append("--threads")
builder.append(shell_quote(str(validate_threads(args.threads, conf))))
if args.endpoint is not None:
builder.append("--endpoint")
builder.append(shell_quote(validate_endpoint(args.endpoint)))
if args.fixed_backoff:
builder.append("--fixed-backoff")
builder.append("run")
start = " ".join(builder)
# Virtualenv support
if hasattr(sys, "real_prefix"):
start = "while [ true ]; do %s; ret=$?; if [ $ret -eq 70 ]; then pip download fishnet || sleep 10; pip install --upgrade fishnet || sleep 10; else exit $ret; fi; sleep 5; done" % start
shell_cmd = "source %s; %s" % (shell_quote(os.path.abspath(os.path.join(sys.prefix, "bin", "activate"))), start)
start = "/bin/bash -c %s" % shell_quote(shell_cmd)
print(template.format(
user=getpass.getuser(),
group=getpass.getuser(),
cwd=os.path.abspath("."),
path=shell_quote(os.environ.get("PATH", "")),
start=start
))
print(file=sys.stderr)
if os.geteuid() == 0:
print("# WARNING: Running as root is not recommended!", file=sys.stderr)
print(file=sys.stderr)
if not hasattr(sys, "real_prefix"):
print("# WARNING: Using a virtualenv (to enable auto update) is recommended!", file=sys.stderr)
print(file=sys.stderr)
print("# Example usage:", file=sys.stderr)
print("# python -m fishnet systemd | sudo tee /etc/systemd/system/fishnet.service", file=sys.stderr)
print("# sudo systemctl enable fishnet.service", file=sys.stderr)
print("# sudo systemctl start fishnet.service", file=sys.stderr)
def install_pyenv(self, versions=None, skip_pkg=False):
"""
We will install pyenv by default. and with normal use lib
Because pyenv is awesome!!!
:param root:
:param skip_pkg: skip install depends package, default is False
:return:
"""
remote_os = get_platform()
if remote_os == 'centos':
if not skip_pkg:
run('yum install -y git')
run("yum install -y gcc gcc-c++ make autoconf certbot"
" libffi-devel ncurses-devel expat-devel"
" zlib-devel zlib libzip-devel"
" bzip2 bzip2-devel bzip2-libs"
" mariadb-devel mariadb-libs"
" sqlite-devel"
" libxml2 libxml2-devel libxslt libxslt-devel"
" libcurl-devel"
" pcre-devel pcre"
" libmcrypt libmcrypt-devel openssl-devel openssl-libs"
" libjpeg libjpeg-devel libpng libpng-devel"
" freetype freetype-devel "
" libtiff-devel lcms2-devel libwebp-devel"
" tcl-devel tk-devel"
)
run('export PYENV_ROOT=/usr/local/var/pyenv && '
'curl -L http://nextoa.com/'
'bin/pyenv-install.sh | bash')
run('ln -sfv /usr/local/var/pyenv/bin/pyenv /usr/local/bin/pyenv')
pass
elif remote_os == 'mac':
with settings(warn_only=True):
run('brew install git pyenv')
pass
run('test -d /usr/local/var/pyenv/plugins/pyenv-virtualenv ||'
' git clone https://github.com/yyuu/pyenv-virtualenv.git'
' /usr/local/var/pyenv/plugins/pyenv-virtualenv')
run('export PYENV_ROOT="/usr/local/var/pyenv/" &&'
' eval "$(pyenv init -)"')
if isinstance(versions, list):
for v in versions:
run('export PYENV_ROOT="/usr/local/var/pyenv/" &&'
' pyenv install -s %s' % v)
elif type(versions).__name__ in ['str', 'unicode']:
run('export PYENV_ROOT="/usr/local/var/pyenv/" &&'
' pyenv install -s %s' % versions)
else:
run('export PYENV_ROOT="/usr/local/var/pyenv/" && '
'pyenv install -s %s' % self.system_python_version)
command_list = [
"""grep "PYENV_ROOT" /etc/profile || \
echo 'export PYENV_ROOT="/usr/local/var/pyenv/"' \
>> /etc/profile""",
"""grep "pyenv init" /etc/profile || \
echo 'eval "$(pyenv init -)"' >> \
/etc/profile""",
"""grep "pyenv virtualenv" /etc/profile || \
echo 'eval "$(pyenv virtualenv-init -)"' \
>> /etc/profile""",
]
if remote_os == 'mac':
command_list = ["sudo sh -c '%s'" % shell_quote(v) for v in
command_list]
pass
for cmd in command_list:
run(cmd)
pass
pass
def run(
self, args, stdin_string=None, env_extend=None, binary_output=False
):
# Allow overriding default settings. If a piece of code really wants to
# set own PATH or CIB_file, we must allow it. I.e. it wants to run
# a pacemaker tool on a CIB in a file but cannot afford the risk of
# changing the CIB in the file specified by the user.
env_vars = self._env_vars.copy()
env_vars.update(
dict(env_extend) if env_extend else dict()
)
log_args = " ".join([shell_quote(x) for x in args])
self._logger.debug(
"Running: {args}\nEnvironment:{env_vars}{stdin_string}".format(
args=log_args,
stdin_string=("" if not stdin_string else (
"\n--Debug Input Start--\n{0}\n--Debug Input End--"
.format(stdin_string)
)),
env_vars=("" if not env_vars else (
"\n" + "\n".join([
" {0}={1}".format(key, val)
for key, val in sorted(env_vars.items())
])
))
)
)
self._reporter.process(
reports.run_external_process_started(
log_args, stdin_string, env_vars
)
)
try:
# pylint: disable=subprocess-popen-preexec-fn
# this is OK as pcs is only single-threaded application
process = subprocess.Popen(
args,
# Some commands react differently if they get anything via stdin
stdin=(
subprocess.PIPE
if stdin_string is not None
else subprocess.DEVNULL
),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=(
lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL)
),
close_fds=True,
shell=False,
env=env_vars,
# decodes newlines and in python3 also converts bytes to str
universal_newlines=(not binary_output)
)
out_std, out_err = process.communicate(stdin_string)
retval = process.returncode
except OSError as e:
raise LibraryError(
reports.run_external_process_error(log_args, e.strerror)
)
self._logger.debug(
(
"Finished running: {args}\nReturn value: {retval}"
+ "\n--Debug Stdout Start--\n{out_std}\n--Debug Stdout End--"
+ "\n--Debug Stderr Start--\n{out_err}\n--Debug Stderr End--"
).format(
args=log_args,
retval=retval,
out_std=out_std,
out_err=out_err
)
)
self._reporter.process(reports.run_external_process_finished(
log_args, retval, out_std, out_err
))
return out_std, out_err, retval
def shell_join(command):
return ' '.join([shell_quote(arg) for arg in command])
def quote(data):
return shell_quote(data)
def convert_rst_to_html_pandoc(input_path_storage=None,
include_bootstrap_css=True,
include_bootstrap_js=False,
include_highlight_js=False,
logger=None):
"""Convert Rst to HTML with Pandoc.
Parameters
----------
input_path_storage : None, optional
Path to where .rst files are stored.
include_bootstrap_css : bool, optional
Whether to include the Bootstrap CSS stylesheet.
include_bootstrap_js : bool, optional
Whether to include the Bootstrap JS script.
include_highlight_js : bool, optional
Whether to include the highlight.js JS script.
logger : LogSystem
The logger.
"""
input_path = file_utils.expand_path(input_path_storage) if input_path_storage else \
os.path.join(PATHS["convertions"], "rst_to_html")
for dirname, dirnames, filenames in os.walk(input_path, topdown=False):
for filename in filenames:
if filename.endswith(".rst") or filename.endswith(".txt"):
f_path = os.path.join(dirname, filename)
f_name = os.path.basename(f_path)
dst_name = os.path.splitext(f_name)[0]
dst_path = os.path.join(os.path.dirname(f_path),
dst_name + ".html")
if file_utils.is_real_file(dst_path):
continue
logger.info(shell_utils.get_cli_separator("-"), date=False)
logger.info("**Converting:**")
logger.info(f_path, date=False)
cmd = [
"pandoc",
shell_quote(f_path), "--output",
shell_quote(dst_path),
"--template=%s" %
shell_quote(PATHS["pandoc_html_template"]), "--wrap=none",
"--no-highlight", "--variable=include-bootstrap-css",
"--variable=include-bootstrap-js",
"--variable=include-highlight-js", "--from=rst",
"--to=html5"
]
try:
logger.info(" ".join(cmd), date=False)
cmd_utils.run_cmd(" ".join(cmd),
stdout=None,
stderr=None,
cwd=os.path.dirname(dst_path),
shell=True,
check=True)
except CalledProcessError as err:
logger.error(err)
def _restart_services(self):
"""Check if the services that are supposed to run on this
machine are actually running. If not, start them.
"""
# To avoid zombies, we poll the process we launched. Anyway we
# use the information from psutil to see if the process we are
# interested in are alive (since if the user has already
# launched another instance, we don't want to duplicate
# services).
new_launched_processes = set([])
for process in self._launched_processes:
if process.poll() is None:
new_launched_processes.add(process)
self._launched_processes = new_launched_processes
# Look for dead processes, and restart them.
matcher = ProcessMatcher()
for service in self._local_services:
# We let the user start logservice and resourceservice.
if service.name == "LogService" or \
service.name == "ResourceService" or \
(self.contest_id is None and
service.name == "ProxyService"):
continue
# If the user specified not to restart some service, we
# ignore it.
if not self._will_restart[service]:
continue
# If we don't have a previously found process, or the one
# we have terminated, we find the process.
proc = self._procs[service]
if proc is None or not proc.is_running():
proc = matcher.find(service, self._services_prev_cpu_times)
self._procs[service] = proc
# If we still do not find it, there is no process, and we
# have nothing to do.
if proc is None or not proc.is_running():
# We give contest_id even if the service doesn't need
# it, since it causes no trouble.
logger.info("Restarting (%s, %s)...", service.name,
service.shard)
command = "cms%s" % service.name
if not config.installed:
command = os.path.join(".", "scripts",
"cms%s" % service.name)
args = [command, "%d" % service.shard]
if self.contest_id is not None:
args += ["-c", str(self.contest_id)]
else:
args += ["-c", "ALL"]
try:
process = subprocess.Popen(args,
stdout=DEVNULL,
stderr=subprocess.STDOUT)
except Exception:
logger.error("Error for command line %s",
shell_quote(" ".join(args)))
raise
self._launched_processes.add(process)
# Run forever.
return True
"""
import unittest
import os
import shlex
import subprocess
import shutil
import time
import sys
try:
from shlex import quote as shell_quote
except ImportError:
from pipes import quote as shell_quote
PYTHON = shell_quote(sys.executable) # make sure to run with same python
class BitBucketTest(unittest.TestCase):
ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
REPO_DIR = os.path.join(ROOT_DIR, 'tests', 'test-git-repo')
SSH_KEY = os.path.join(ROOT_DIR, 'tests', 'ssh-forward-proxy-test-key')
PROXY_PATH = os.path.join(ROOT_DIR, 'bin', 'ssh-forward-proxy.py')
BITBUCKET = 'bitbucket.org'
REPO_URL = '[email protected]:lincheney/ssh-forward-proxy-test.git'
GIT_PATH = os.path.join(REPO_DIR, '.git')
README_PATH = os.path.join(REPO_DIR, 'README')
README_TEXT = 'This is a test for ssh-forward-proxy.\n'
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
}
}
NODE_ROOT = join(PROJECT_DIR, '..', 'node_modules')
HANDLEBARS_PATH = join(NODE_ROOT, 'django-ember-precompile', 'bin', 'django-ember-precompile')
# Escape characters in the file path (like spaces)
# Prevents django-compress manipulation errors
HANDLEBARS_PATH = shell_quote(HANDLEBARS_PATH)
COMPRESS_PRECOMPILERS = (
('text/x-handlebars', '{} {{infile}}'.format(HANDLEBARS_PATH)),
)
REST_FRAMEWORK = {
'FILTER_BACKEND': 'rest_framework.filters.DjangoFilterBackend'
}
def input(self, **kwargs):
encoding = self.default_encoding
options = dict(self.options)
if self.infile is None and "{infile}" in self.command:
# create temporary input file if needed
if self.filename is None:
self.infile = NamedTemporaryFile(mode='wb')
self.infile.write(self.content.encode(encoding))
self.infile.flush()
options["infile"] = self.infile.name
else:
# we use source file directly, which may be encoded using
# something different than utf8. If that's the case file will
# be included with charset="something" html attribute and
# charset will be available as filter's charset attribute
encoding = self.charset # or self.default_encoding
self.infile = open(self.filename)
options["infile"] = self.filename
if "{outfile}" in self.command and "outfile" not in options:
# create temporary output file if needed
ext = self.type and ".%s" % self.type or ""
self.outfile = NamedTemporaryFile(mode='r+', suffix=ext)
options["outfile"] = self.outfile.name
# Quote infile and outfile for spaces etc.
if "infile" in options:
options["infile"] = shell_quote(options["infile"])
if "outfile" in options:
options["outfile"] = shell_quote(options["outfile"])
try:
command = self.command.format(**options)
proc = subprocess.Popen(command,
shell=True,
cwd=self.cwd,
stdout=self.stdout,
stdin=self.stdin,
stderr=self.stderr)
if self.infile is None:
# if infile is None then send content to process' stdin
filtered, err = proc.communicate(self.content.encode(encoding))
else:
filtered, err = proc.communicate()
filtered, err = filtered.decode(encoding), err.decode(encoding)
except (IOError, OSError) as e:
raise FilterError('Unable to apply %s (%r): %s' %
(self.__class__.__name__, self.command, e))
else:
if proc.wait() != 0:
# command failed, raise FilterError exception
if not err:
err = ('Unable to apply %s (%s)' %
(self.__class__.__name__, self.command))
if filtered:
err += '\n%s' % filtered
raise FilterError(err)
if self.verbose:
self.logger.debug(err)
outfile_path = options.get('outfile')
if outfile_path:
with io.open(outfile_path, 'r', encoding=encoding) as file:
filtered = file.read()
finally:
if self.infile is not None:
self.infile.close()
if self.outfile is not None:
self.outfile.close()
return smart_text(filtered)
def _restart_services(self):
"""Check if the services that are supposed to run on this
machine are actually running. If not, start them.
"""
# To avoid zombies, we poll the process we launched. Anyway we
# use the information from psutil to see if the process we are
# interested in are alive (since if the user has already
# launched another instance, we don't want to duplicate
# services).
new_launched_processes = set([])
for process in self._launched_processes:
if process.poll() is None:
new_launched_processes.add(process)
self._launched_processes = new_launched_processes
# Look for dead processes, and restart them.
matcher = ProcessMatcher()
for service in self._local_services:
# We let the user start logservice and resourceservice.
if service.name == "LogService" or \
service.name == "ResourceService" or \
(self.contest_id is None and
service.name == "ProxyService"):
continue
# If the user specified not to restart some service, we
# ignore it.
if not self._will_restart[service]:
continue
# If we don't have a previously found process, or the one
# we have terminated, we find the process.
proc = self._procs[service]
if proc is None or not proc.is_running():
proc = matcher.find(service, self._services_prev_cpu_times)
self._procs[service] = proc
# If we still do not find it, there is no process, and we
# have nothing to do.
if proc is None or not proc.is_running():
# We give contest_id even if the service doesn't need
# it, since it causes no trouble.
logger.info("Restarting (%s, %s)...",
service.name, service.shard)
command = os.path.join(BIN_PATH, "cms%s" % service.name)
if not config.installed:
command = os.path.join(
".",
"scripts",
"cms%s" % service.name)
args = [command, "%d" % service.shard]
if self.contest_id is not None:
args += ["-c", str(self.contest_id)]
else:
args += ["-c", "ALL"]
try:
process = subprocess.Popen(args,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
except Exception:
logger.error("Error for command line %s",
shell_quote(" ".join(args)))
raise
self._launched_processes.add(process)
# Run forever.
return True
def run(
self,
args: Sequence[str],
stdin_string: Optional[str] = None,
env_extend: Optional[Mapping[str, str]] = None,
binary_output: bool = False,
) -> Tuple[str, str, int]:
# Allow overriding default settings. If a piece of code really wants to
# set own PATH or CIB_file, we must allow it. I.e. it wants to run
# a pacemaker tool on a CIB in a file but cannot afford the risk of
# changing the CIB in the file specified by the user.
env_vars = dict(self._env_vars)
env_vars.update(dict(env_extend) if env_extend else {})
log_args = " ".join([shell_quote(x) for x in args])
self._logger.debug(
"Running: {args}\nEnvironment:{env_vars}{stdin_string}".format(
args=log_args,
stdin_string=("" if not stdin_string else (
"\n--Debug Input Start--\n{0}\n--Debug Input End--"
).format(stdin_string)),
env_vars=("" if not env_vars else ("\n" + "\n".join([
" {0}={1}".format(key, val)
for key, val in sorted(env_vars.items())
]))),
))
self._reporter.report(
ReportItem.debug(
reports.messages.RunExternalProcessStarted(
log_args,
stdin_string,
env_vars,
)))
try:
# pylint: disable=subprocess-popen-preexec-fn, consider-using-with
# this is OK as pcs is only single-threaded application
process = subprocess.Popen(
args,
# Some commands react differently if they get anything via stdin
stdin=(subprocess.PIPE
if stdin_string is not None else subprocess.DEVNULL),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=(
lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL)),
close_fds=True,
shell=False,
env=env_vars,
# decodes newlines and in python3 also converts bytes to str
universal_newlines=(not binary_output),
)
out_std, out_err = process.communicate(stdin_string)
retval = process.returncode
except OSError as e:
raise LibraryError(
ReportItem.error(
reports.messages.RunExternalProcessError(
log_args,
e.strerror,
))) from e
self._logger.debug(
("Finished running: {args}\nReturn value: {retval}" +
"\n--Debug Stdout Start--\n{out_std}\n--Debug Stdout End--" +
"\n--Debug Stderr Start--\n{out_err}\n--Debug Stderr End--"
).format(args=log_args,
retval=retval,
out_std=out_std,
out_err=out_err))
self._reporter.report(
ReportItem.debug(
reports.messages.RunExternalProcessFinished(
log_args,
retval,
out_std,
out_err,
)))
return out_std, out_err, retval
