def test_universal1(self):
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;' + SETBINARY +
'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
'sys.stdout.flush();'
'sys.stdout.write("line3\\r\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line4\\r");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline5");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'],
stdout=subprocess.PIPE,
universal_newlines=1,
bufsize=1)
try:
stdout = p.stdout.read()
if PY3 and isinstance(stdout, bytes):
# OS X gives us binary back from stdout.read, but linux (travis ci)
# gives us text...text is correct because we're in universal newline
# mode
stdout = stdout.decode('ascii')
if python_universal_newlines:
# Interpreter with universal newline support
self.assertEqual(stdout,
"line1\nline2\nline3\nline4\nline5\nline6")
else:
# Interpreter without universal newline support
self.assertEqual(stdout,
"line1\nline2\rline3\r\nline4\r\nline5\nline6")
finally:
p.stdout.close()
def test_universal2(self):
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;' + SETBINARY +
'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
'sys.stdout.flush();'
'sys.stdout.write("line3\\r\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line4\\r\\nline5");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'],
stdout=subprocess.PIPE,
universal_newlines=1,
bufsize=1)
try:
stdout = p.stdout.read()
if python_universal_newlines:
# Interpreter with universal newline support
if not python_universal_newlines_broken:
self.assertEqual(stdout,
"line1\nline2\nline3\nline4\nline5\nline6")
else:
# Note the extra newline after line 3
self.assertEqual(stdout,
'line1\nline2\nline3\n\nline4\n\nline5\nline6')
else:
# Interpreter without universal newline support
self.assertEqual(stdout,
"line1\nline2\rline3\r\nline4\r\nline5\nline6")
finally:
p.stdout.close()
def test_issue148(self):
for _ in range(7):
with self.assertRaises(OSError) as exc:
with subprocess.Popen('this_name_must_not_exist'):
pass
self.assertEqual(exc.exception.errno, errno.ENOENT)
def ffmpeg_cut_segment(segment, cut_start=None, cut_end=None):
"""Return a Popen object which is ffmpeg cutting the given single segment.
This is used when doing a fast cut.
"""
args = [
'ffmpeg',
'-hide_banner',
'-loglevel',
'error', # suppress noisy output
'-i',
segment.path,
]
# output from ffprobe is generally already sorted but let's be paranoid,
# because the order of map args matters.
for stream in sorted(streams_info(segment),
key=lambda stream: stream['index']):
# map the same stream in the same position from input to output
args += ['-map', '0:{}'.format(stream['index'])]
if stream['codec_type'] in ('video', 'audio'):
# for non-metadata streams, make sure we use the same codec (metadata streams
# are a bit weirder, and ffmpeg will do the right thing anyway)
args += ['-codec:{}'.format(stream['index']), stream['codec_name']]
# now add trim args
if cut_start:
args += ['-ss', str(cut_start)]
if cut_end:
args += ['-to', str(cut_end)]
# output to stdout as MPEG-TS
args += ['-f', 'mpegts', '-']
# run it
logging.info("Running segment cut with args: {}".format(" ".join(args)))
return subprocess.Popen(args, stdout=subprocess.PIPE)
def mv_pay_pic(src_file_path, tgt_file_name, tgt_file_path):
"""
移动支付截图文件，从存储的地方移动到下载地方
:param src_file_path: 源文件路径
:param tgt_file_name: 目的文件名称
:param tgt_file_path: 目的文件路径
:return:
"""
if not os.path.exists(tgt_file_path):
os.makedirs(tgt_file_path)
tgt_path = "%s/%s" % (tgt_file_path, tgt_file_name)
if platform.system() == 'Linux':
mv_cmd = "sudo mv %s %s" % (src_file_path, tgt_path)
# 如果图片存储主机不是本机，则需要ssh远程连接，同时需要能实现ssh无密码登录
if ArgumentParser().args.pic_store_ip != "localhost":
mv_cmd = "ssh -p%s %s@%s %s" % (
ArgumentParser().args.pic_store_port,
ArgumentParser().args.pic_store_user,
ArgumentParser().args.pic_store_ip, mv_cmd)
subprocess.Popen(mv_cmd, shell=True)
logger.info("mv_pay_pic src_file_path:%s tgt_path:%s, mv_cmd:%s" %
(src_file_path, tgt_path, mv_cmd))
def _popen(self, command,
stdin=None, stdout=None, stderr=None,
close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (int|None): a file descriptor.
stdout (int|None): a file descriptor.
stderr (int|None): a file descriptor.
close_fds (bool): close all file descriptor before executing.
return (Popen): popen object.
"""
self.exec_num += 1
self.log = None
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
try:
p = subprocess.Popen(args,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
except OSError:
logger.critical("Failed to execute program in sandbox "
"with command: %s", pretty_print_cmdline(args),
exc_info=True)
raise
return p
def test_universal2(self):
p = subprocess.Popen([
sys.executable, "-c",
'import sys,os;' + SETBINARY + 'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
'sys.stdout.flush();'
'sys.stdout.write("line3\\r\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line4\\r\\nline5");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'
],
stdout=subprocess.PIPE,
universal_newlines=1)
try:
stdout = p.stdout.read()
if hasattr(file, 'newlines'):
# Interpreter with universal newline support
self.assertEqual(stdout,
"line1\nline2\nline3\nline4\nline5\nline6")
else:
# Interpreter without universal newline support
self.assertEqual(
stdout, "line1\nline2\rline3\r\nline4\r\nline5\nline6")
finally:
p.stdout.close()
def spawn_subproc(p):
proc = subprocess.Popen(
"export CAMS_YML=/tmp/cams_bench_{}.yml && python run.py".
format(p),
shell=True,
preexec_fn=os.setsid)
procs.append(proc)
def _run(self):
try:
from subprocess import DEVNULL
except ImportError:
import os
DEVNULL = open(os.devnull, 'wb')
try:
prog = self._args[5].split('/')[-1]
while self._keep_running:
logging.info('Starting new %s process...', prog)
self._process = subprocess.Popen(self._args,
stdout=DEVNULL,
stderr=subprocess.STDOUT)
return_code = self._process.wait()
logging.info(
'Interference application %s exited with return code %d',
self._application_name, return_code)
# Return code of -9 means SIGKILL, this is ok
if not (return_code == 0 or return_code == -9):
self._keep_running = False
raise Exception('Process failure, return code %d' %
(return_code))
except Exception as e:
logging.exception('Interference application %s failed',
self._application_name)
raise
self._process = None
def add_agent_user(self, agent_name, agent_dir):
"""
Invokes sudo to create a unique unix user for the agent.
:param agent_name:
:param agent_dir:
:return:
"""
# Ensure the agent users unix group exists
self.add_agent_user_group()
# Create a USER_ID file, truncating existing USER_ID files which
# should at this point be considered unsafe
user_id_path = os.path.join(agent_dir, "USER_ID")
with open(user_id_path, "w+") as user_id_file:
volttron_agent_user = "******".format(
str(get_utc_seconds_from_epoch()).replace(".", ""))
_log.info("Creating volttron user {}".format(volttron_agent_user))
group = "volttron_{}".format(self.instance_name)
useradd = ['sudo', 'useradd', volttron_agent_user, '-r', '-G', group]
useradd_process = subprocess.Popen(
useradd, stdout=PIPE, stderr=PIPE)
stdout, stderr = useradd_process.communicate()
if useradd_process.returncode != 0:
# TODO alert?
raise RuntimeError("Creating {} user failed: {}".format(
volttron_agent_user, stderr))
user_id_file.write(volttron_agent_user)
return volttron_agent_user
def run_local_command(self,
command,
output_queue=None,
output_filter=None,
raise_error=False):
"""Run a shell command on this machine without blocking
Arguments:
command - command to run
output_queue - passed to process_output if given
output_filter - passed to process_output
raise_error - passed to process_output
"""
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if output_queue:
return gevent.spawn(self.process_output,
process.stdout,
output_queue,
output_filter=output_filter,
process=process,
raise_error=raise_error)
return process
def hook(self, name, *args):
"""
Executes a given hook. All additional arguments are passed to the
hook as script arguments.
:param name: Hook name (like session.pre-up)
"""
script = self.hooks.get(name, None)
if not script:
return
# Execute the registered hook
logger.debug("Executing hook '%s' via script '%s %s'." % (name, script, str([str(x) for x in args])))
try:
script_process = gevent_subprocess.Popen([script] + [str(x) for x in args],
stdout=gevent_subprocess.PIPE,
stderr=gevent_subprocess.PIPE)
stdout, stderr = script_process.communicate()
if stdout:
logger.debug("Hook '%s' script stdout:" % script)
logger.debug(stdout)
if stderr:
logger.debug("Hook '%s' script stderr:" % script)
logger.warning(stderr)
except:
logger.warning("Failed to execute hook '%s'!" % script)
logger.warning(traceback.format_exc())
def test_child_exception(self):
try:
subprocess.Popen(['*']).wait()
except OSError as ex:
assert ex.errno == 2, ex
else:
raise AssertionError('Expected OSError: [Errno 2] No such file or directory')
def test_agent_filters(volttron_instance):
auuid = volttron_instance.install_agent(
agent_dir=get_examples("ListenerAgent"), start=True)
buuid = volttron_instance.install_agent(
agent_dir=get_examples("ListenerAgent"), start=True)
# Verify all installed agents show up in list
with with_os_environ(volttron_instance.env):
p = subprocess.Popen(
["volttron-ctl", "list"],
env=volttron_instance.env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
agent_list = p.communicate()
assert "listeneragent-3.3_1" in str(agent_list)
assert "listeneragent-3.3_2" in str(agent_list)
# Filter agent based on agent uuid
with with_os_environ(volttron_instance.env):
p = subprocess.Popen(
["volttron-ctl", "list", str(auuid)],
env=volttron_instance.env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
agent_list = p.communicate()
assert "listeneragent-3.3_1" in str(agent_list)
assert "listeneragent-3.3_2" not in str(agent_list)
# Filter agent based on agent name
with with_os_environ(volttron_instance.env):
p = subprocess.Popen(
["volttron-ctl", "list", "listeneragent-3.3_1"],
env=volttron_instance.env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
agent_list = p.communicate()
assert "listeneragent-3.3_1" in str(agent_list)
assert "listeneragent-3.3_2" not in str(agent_list)
def main():
print('Running on Python %s' % sys.version)
parser = argparse.ArgumentParser()
parser.add_argument(
'--data-root',
action='store',
default='data',
help='Location of the data dir containing all config files and logs.')
args = parser.parse_args()
data_root = args.data_root
config = configparser.ConfigParser()
with open(get_shared_ini_path(data_root)) as f:
config.read_file(f)
ports = Ports(int(config['shared']['port_offset']))
try:
udp_proxy_proc1 = sp.Popen('udpproxy.exe %d' % ports['gameserver1'])
udp_proxy_proc2 = sp.Popen('udpproxy.exe %d' % ports['gameserver2'])
except OSError as e:
print(
'Failed to run udpproxy.exe. Run download_udpproxy.py to download it\n'
'or build it yourself using the Visual Studio solution in the udpproxy\n'
'subdirectory and place it in the taserver directory.\n',
file=sys.stderr)
return
server_queue = gevent.queue.Queue()
firewall = Firewall(ports, data_root)
gevent_spawn('firewall.run', firewall.run, server_queue)
def handle_client(socket, address):
msg = TcpMessageReader(socket).receive()
command = json.loads(msg.decode('utf8'))
server_queue.put(command)
server = StreamServer(('127.0.0.1', ports['firewall']), handle_client)
try:
server.serve_forever()
except KeyboardInterrupt:
firewall.remove_all_rules()
udp_proxy_proc1.terminate()
udp_proxy_proc2.terminate()
def execute(self, *args, **kwargs):
try:
self.env = kwargs.get('env', None)
self.process = subprocess.Popen(*args, **kwargs)
except OSError as e:
if e.filename:
raise
raise OSError(*(e.args + (args[0], )))
def _git(self, cmd, stdin=None):
"""Executes a git command and returns the standard output."""
p = subprocess.Popen(['git'] + cmd, cwd=self._repo, stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, _ = p.communicate(stdin)
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, ['git'] + cmd, None)
return stdout.strip().splitlines()
def test_leak(self):
num_before = greentest.get_number_open_files()
p = subprocess.Popen([sys.executable, "-c", "print()"],
stdout=subprocess.PIPE)
p.wait()
del p
num_after = greentest.get_number_open_files()
self.assertEqual(num_before, num_after)
def __test_no_output(self, kwargs, kind):
proc = subprocess.Popen([sys.executable, '-c', 'pass'],
stdout=subprocess.PIPE,
**kwargs)
stdout, stderr = proc.communicate()
self.assertIsInstance(stdout, kind)
self.assertIsNone(stderr)
def test_issue148(self):
for i in range(7):
try:
subprocess.Popen('this_name_must_not_exist')
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
else:
raise AssertionError('must fail with ENOENT')
def create_process(self):
self.status = 0 #starting
cwd = self.old_cwd.rstrip('>')
self.process = sub.Popen(self.cmd_args,
stdin=sub.PIPE,
stdout=sub.PIPE,
stderr=sub.STDOUT,
shell=False,
cwd=cwd)
def call(args):
if USE_SU:
proc = subprocess.Popen('su', stdin=subprocess.PIPE)
proc.stdin.write(' '.join(args))
proc.stdin.write('\nexit\n')
proc.communicate()
return proc.poll()
else:
return subprocess.call(args)
def sudo_kill(name):
pid = find_real_pid(name)
if pid:
LOGGER.info('kill %s' % pid)
proc = subprocess.Popen('su', stdin=subprocess.PIPE)
proc.stdin.write('exec '+ BUSYBOX_PATH +' kill %s\n' % pid)
proc.communicate()
else:
LOGGER.info('%s not found in /proc' % name)
def _restart_services(self):
"""Check if the services that are supposed to run on this
machine are actually running. If not, start them.
"""
# To avoid zombies, we poll the process we launched. Anyway we
# use the information from psutil to see if the process we are
# interested in are alive (since if the user has already
# launched another instance, we don't want to duplicate
# services).
new_launched_processes = set([])
for process in self._launched_processes:
if process.poll() is None:
new_launched_processes.add(process)
self._launched_processes = new_launched_processes
# Look for dead processes, and restart them.
matcher = ProcessMatcher()
for service in self._local_services:
# We let the user start logservice and resourceservice.
if service.name == "LogService" or \
service.name == "ResourceService":
continue
# If the user specified not to restart some service, we
# ignore it.
if not self._will_restart[service]:
continue
# If we don't have a previously found process, or the one
# we have terminated, we find the process.
proc = self._procs[service]
if proc is None or not proc.is_running():
proc = matcher.find(service, self._services_prev_cpu_times)
self._procs[service] = proc
# If we still do not find it, there is no process, and we
# have nothing to do.
if proc is None or not proc.is_running():
# We give contest_id even if the service doesn't need
# it, since it causes no trouble.
logger.info("Restarting (%s, %s)...", service.name,
service.shard)
command = "cms%s" % service.name
if not config.installed:
command = os.path.join(".", "scripts",
"cms%s" % service.name)
process = subprocess.Popen([
command,
"%d" % service.shard, "-c",
"%d" % self.contest_id
],
stdout=DEVNULL,
stderr=subprocess.STDOUT)
self._launched_processes.add(process)
# Run forever.
return True
def sudo_kill(name):
pid = find_real_pid(name)
if pid:
LOGGER.info('kill %s' % pid)
proc = subprocess.Popen('su', stdin=subprocess.PIPE)
proc.stdin.write('exec /data/data/fq.router2/busybox kill %s\n' % pid)
proc.communicate()
else:
LOGGER.info('%s not found in /proc' % name)
def _start_and_register_worker(self):
port = random.randint(5000, 6000)
endpoint = "tcp://127.0.0.1:{}".format(port)
cmd = "python3 worker.py {}".format(endpoint)
proc = subprocess.Popen(cmd, shell=True)
client = zerorpc.Client(endpoint, timeout=100000, heartbeat=None)
worker = Worker(port, proc, client)
self.workers[worker.id] = worker
print("started worker {}".format(port))
def _install_protos(proto_package_path, dgst, proto_files):
"""Installs protos to `{proto_package_path}/PB`.
Args:
* proto_package_base (str) - The absolute path to the folder where:
* We should install protoc as '.../protoc/...'
* We should install the compiled proto files as '.../PB/...'
* We should use '.../tmp/...' as a tempdir.
* dgst (str) - The hexadecimal (lowercase) checksum for the protos we're
about to install.
* proto_files (List[Tuple[src_abspath: str, dest_relpath: str]])
Side-effects:
* Ensures that `{proto_package_path}/PB` exists and is the correct
version (checksum).
* Ensures that `{proto_package_path}/protoc` contains the correct
`protoc` compiler from CIPD.
"""
cipd_proc = subprocess.Popen([
'cipd' + _BAT, 'ensure', '-root',
os.path.join(proto_package_path, 'protoc'), '-ensure-file', '-'
],
stdin=subprocess.PIPE)
cipd_proc.communicate('''
infra/tools/protoc/${{platform}} protobuf_version:v{PROTOC_VERSION}
'''.format(PROTOC_VERSION=PROTOC_VERSION))
if cipd_proc.returncode != 0:
raise ValueError('failed to install protoc: retcode %d' %
cipd_proc.returncode)
# This tmp folder is where all the temporary garbage goes. Future recipe
# engine invocations will attempt to clean this up as long as PB is
# up-to-date.
tmp_base = os.path.join(proto_package_path, 'tmp')
# proto_tree holds a tree of all the collected .proto files, to be passed to
# `protoc`
# pb_temp is the destination of all the generated files; it will be renamed to
# `{proto_package_path}/dest` as the final step of the installation.
_DirMaker()(tmp_base)
proto_tree = tempfile.mkdtemp(dir=tmp_base)
pb_temp = tempfile.mkdtemp(dir=tmp_base)
argfile_fd, argfile = tempfile.mkstemp(dir=tmp_base)
_collect_protos(argfile_fd, proto_files, proto_tree)
protoc = os.path.join(proto_package_path, 'protoc', 'protoc')
_compile_protos(proto_files, proto_tree, protoc, argfile, pb_temp)
with open(os.path.join(pb_temp, 'csum'), 'wb') as csum_f:
csum_f.write(dgst)
dest = os.path.join(proto_package_path, 'PB')
# Check the digest again, in case another engine beat us to the punch.
# This is still racy, but it makes the window substantially smaller.
if not _check_digest(proto_package_path, dgst):
old = tempfile.mkdtemp(dir=tmp_base)
_try_rename(dest, os.path.join(old, 'PB'))
_try_rename(pb_temp, dest)
def start_server_process(self, server):
external_port = self.ports[server]
internal_port = self.ports[f'{server}proxy']
log_filename = os.path.join(self.get_my_documents_folder(), 'My Games',
'Tribes Ascend', 'TribesGame', 'Logs',
'tagameserver%d.log' % external_port)
try:
self.logger.info(
f'{server}: removing previous log file {log_filename}')
os.remove(log_filename)
except FileNotFoundError:
pass
self.logger.info(
f'{server}: starting a new TribesAscend server on port {external_port}...'
)
# Add 100 to the port, because it's the udpproxy that's actually listening on the port itself
# and it forwards traffic to port + 100
args = [
self.exe_path, 'server',
'-Log=tagameserver%d.log' % external_port,
'-port=%d' % internal_port, '-controlport',
str(self.ports['game2launcher'])
]
if self.dll_config_path is not None:
args.extend(['-tamodsconfig', self.dll_config_path])
process = sp.Popen(args, cwd=self.working_dir)
self.servers[server] = process
self.logger.info(f'{server}: started process with pid {process.pid}')
# Check if it doesn't exit right away
time.sleep(2)
ret_code = process.poll()
if ret_code:
raise FatalError(
'The game server process terminated almost immediately with exit code %08X'
% ret_code)
self.logger.info(
f'{server}: waiting until game server has finished starting up...')
if not self.wait_until_file_contains_string(
log_filename, 'Log: Bringing up level for play took:',
timeout=30):
self.logger.warning(
f'{server}: timeout waiting for log entry, continuing with injection...'
)
self.logger.info(
f'{server}: injecting game controller DLL into game server...')
inject(process.pid, self.dll_to_inject)
self.logger.info(f'{server}: injection done.')
self.watcher_task = gevent_spawn(
'gameserver process watcher for server %s' % server,
self.server_process_watcher, process, server)
def start(self):
"""
Reimplements Executor and SimpleExecutor start to allow setting stdin/stdout/stderr/cwd
It may break input/output/communicate, but will ensure child output redirects won't
break parent process by filling the PIPE.
Also, catches ProcessExitedWithError and raise FileNotFoundError if exitcode was 127
"""
if self.pre_start_check():
# Some other executor (or process) is running with same config:
raise AlreadyRunning(self)
if self.process is None:
command = self.command
if not self._shell:
command = self.command_parts # type: ignore
if isinstance(self.stdio, (list, tuple)):
stdin, stdout, stderr = self.stdio
else:
stdin = stdout = stderr = self.stdio # type: ignore
env = os.environ.copy()
env[ENV_UUID] = self._uuid
popen_kwargs = {
"shell": self._shell,
"stdin": stdin,
"stdout": stdout,
"stderr": stderr,
"universal_newlines": True,
"env": env,
"cwd": self.cwd,
}
if platform.system() != "Windows":
popen_kwargs["preexec_fn"] = os.setsid # type: ignore
self.process = subprocess.Popen(command, **popen_kwargs)
self._set_timeout()
try:
self.wait_for(self.check_subprocess)
except ProcessExitedWithError as e:
if e.exit_code == 127:
raise FileNotFoundError(
f"Can not execute {command!r}, check that the executable exists."
) from e
else:
output_file_names = {
io.name
for io in (stdout, stderr)
if hasattr(io, "name") # type: ignore
}
if output_file_names:
log.warning("Process output file(s)",
output_files=output_file_names)
raise
return self
def _mock_popen(self, rcpt, returncode, stdout):
pmock = self.mox.CreateMock(subprocess.Popen)
subprocess.Popen(['relaytest', '-f', '*****@*****.**', rcpt],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).AndReturn(pmock)
pmock.communicate(b'From: [email protected]\r\n\r\ntest test\r\n').AndReturn((stdout, ''))
pmock.pid = -1
pmock.returncode = returncode
return pmock