def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Update a repository. Return False if missing; return True if successful; raise an exception if updating fails.""" repo.is_dir() repo_type = typeOfRepo(repo) if repo_type == "hg": hg_pull_cmd = ["hg", "--time", "pull", "-u"] logger.info("\nRunning `%s` now..\n", " ".join(hg_pull_cmd)) out_hg_pull = subprocess.run(hg_pull_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_pull.args), out_hg_pull.stderr) hg_log_default_cmd = ["hg", "--time", "log", "-r", "default"] logger.info("\nRunning `%s` now..\n", " ".join(hg_log_default_cmd)) out_hg_log_default = subprocess.run(hg_log_default_cmd, check=True, cwd=str(repo), stderr=subprocess.PIPE) logger.info('"%s" had the above output and took - %s', subprocess.list2cmdline(out_hg_log_default.args), out_hg_log_default.stderr) elif repo_type == "git": # Ignore exit codes so the loop can continue retrying up to number of counts. gitenv = deepcopy(os.environ) if platform.system() == "Windows": gitenv["GIT_SSH_COMMAND"] = "~/../../mozilla-build/msys/bin/ssh.exe -F ~/.ssh/config" time_cmd([GITBINARY, "pull"], cwd=str(repo), env=gitenv) else: raise Exception("Unknown repository type: " + repo_type) return True
def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Update a repository. Return False if missing; return True if successful; raise an exception if updating fails.""" assert os.path.isdir(repo) repo_type = typeOfRepo(repo) if repo_type == 'hg': out_hg_pull = subprocess.run(["hg", "--time", "pull", "-u"], check=True, cwd=repo, stderr=subprocess.DEVNULL) logger.info("Command with the above out_hg_pull: %s", subprocess.list2cmdline(out_hg_pull.args)) out_hg_log_default = subprocess.run( ["hg", "--time", "log", "-r", "default"], check=True, cwd=repo, stderr=subprocess.DEVNULL) logger.info("Command with the above output: %s", subprocess.list2cmdline(out_hg_log_default.args)) elif repo_type == 'git': # Ignore exit codes so the loop can continue retrying up to number of counts. gitenv = deepcopy(os.environ) if sps.isWin: gitenv[ 'GIT_SSH_COMMAND'] = "~/../../mozilla-build/msys/bin/ssh.exe -F ~/.ssh/config" time_cmd([GITBINARY, "pull"], cwd=repo, env=gitenv, stderr=subprocess.DEVNULL) else: raise Exception('Unknown repository type: ' + repo_type) return True
def __enter__(self): monitor_path = join(self.tempdir, 'monitor') if not self.boot: serial_path = join(self.tempdir, 'serial') qemu_args = [self.qemu_path, self.qcow] # if not self.boot: # qemu_args.extend( ['-monitor', 'unix:{},server,nowait'.format(monitor_path)]) if self.boot: qemu_args.append('-S') else: qemu_args.extend([ '-serial', 'unix:{},server,nowait'.format(serial_path), '-loadvm', self.snapshot ]) qemu_args.extend(['-display', 'none']) qemu_args.extend(self.extra_args) if self.rr: qemu_args = ['rr', 'record'] + qemu_args if self.perf: qemu_args = ['perf', 'record'] + qemu_args progress("Running qemu with args:") print subprocess32.list2cmdline(qemu_args) self.qemu = subprocess32.Popen( qemu_args) # , stdout=DEVNULL, stderr=DEVNULL) while not os.path.exists(monitor_path): time.sleep(0.1) if not self.boot: while not os.path.exists(serial_path): time.sleep(0.1) # while not all([os.path.exists(p) for p in [monitor_path, serial_path]]): # time.sleep(0.1) self.monitor_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.monitor_socket.connect(monitor_path) self.monitor = Expect(self.monitor_socket) if not self.boot: self.serial_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.serial_socket.connect(serial_path) self.console = Expect(self.serial_socket) # Make sure monitor/console are in right state. self.monitor.expect("(qemu)") print if not self.boot: self.console.sendline() self.console.expect(self.expect_prompt) print print return self
def __enter__(self): monitor_path = join(self.tempdir, 'monitor') if not self.boot: serial_path = join(self.tempdir, 'serial') qemu_args = [self.qemu_path, self.qcow] # if not self.boot: # qemu_args.extend(['-monitor', 'unix:{},server,nowait'.format(monitor_path)]) if self.boot: qemu_args.append('-S') else: qemu_args.extend(['-serial', 'unix:{},server,nowait'.format(serial_path), '-loadvm', self.snapshot]) qemu_args.extend(['-display', 'none']) qemu_args.extend(self.extra_args) if self.rr: qemu_args = ['rr', 'record'] + qemu_args if self.perf: qemu_args = ['perf', 'record'] + qemu_args progress("Running qemu with args:") print subprocess32.list2cmdline(qemu_args) self.qemu = subprocess32.Popen(qemu_args) # , stdout=DEVNULL, stderr=DEVNULL) while not os.path.exists(monitor_path): time.sleep(0.1) if not self.boot: while not os.path.exists(serial_path): time.sleep(0.1) # while not all([os.path.exists(p) for p in [monitor_path, serial_path]]): # time.sleep(0.1) self.monitor_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.monitor_socket.connect(monitor_path) self.monitor = Expect(self.monitor_socket) if not self.boot: self.serial_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.serial_socket.connect(serial_path) self.console = Expect(self.serial_socket) # Make sure monitor/console are in right state. self.monitor.expect("(qemu)") print if not self.boot: self.console.sendline() self.console.expect(self.expect_prompt) print print return self
def create_recording(qemu_path, qcow, snapshot, command, copy_directory, recording_path, isoname=None, rr=False): DEVNULL = open(os.devnull, "w") recording_path = realpath(recording_path) if not isoname: isoname = copy_directory + '.iso' with TempDir() as tempdir, Qemu(qemu_path, qcow, snapshot, tempdir, rr=rr) as qemu: if os.listdir(copy_directory): progress("Creating ISO {}...".format(isoname)) make_iso(copy_directory, isoname) progress("Inserting CD...") qemu.run_monitor("change ide1-cd0 \"{}\"".format(isoname)) qemu.run_console("mkdir -p {}".format(pipes.quote(copy_directory))) # Make sure cdrom didn't automount # Make sure guest path mirrors host path qemu.run_console("while ! mount /dev/cdrom {}; ".format(pipes.quote(copy_directory)) + "do sleep 0.3; umount /dev/cdrom; done") # Important that we type command into console before recording starts and only # hit enter once we've started the recording. progress("Running command inside guest.") qemu.type_console(subprocess32.list2cmdline(command)) # start PANDA recording qemu.run_monitor("begin_record \"{}\"".format(recording_path)) qemu.run_console(timeout=1200) # end PANDA recording progress("Ending recording...") qemu.run_monitor("end_record") DEVNULL.close()
def install(self, filepath): ''' TODO(ssx): not tested. ''' if not os.path.exists(filepath): raise EnvironmentError('file "%s" not exists.' % filepath) ideviceinstaller = must_look_exec('ideviceinstaller') os.system(subprocess.list2cmdline([ideviceinstaller, '-u', self.udid, '-i', filepath]))
def __init__(self, executable='ffmpeg', global_options='', inputs=None, outputs=None): """Initialize FFmpeg command line wrapper. Compiles FFmpegg command line from passed arguments (executable path, options, inputs and outputs). For more info about FFmpeg command line format see `here <https://ffmpeg.org/ffmpeg.html#Synopsis>`_. :param str executable: path to ffmpeg executable; by default the ``ffmpeg`` command will be searched for in the ``PATH``, but can be overridden with an absolute path to ``ffmpeg`` executable :param iterable global_options: global options passed to ``ffmpeg`` executable (e.g. ``-y``, ``-v`` etc.); can be specified either as a list/tuple/set of strings, or one space-separated string; by default no global options are passed :param dict inputs: a dictionary specifying one or more input arguments as keys with their corresponding options (either as a list of strings or a single space separated string) as values :param dict outputs: a dictionary specifying one or more output arguments as keys with their corresponding options (either as a list of strings or a single space separated string) as values """ self.executable = executable self._cmd = [executable] if not _is_sequence(global_options): global_options = shlex.split(global_options) self._cmd += global_options self._cmd += _merge_args_opts(inputs, add_input_option=True) self._cmd += _merge_args_opts(outputs) self.cmd = subprocess.list2cmdline(self._cmd)
def must_exec(*cmds, **kwargs): prompt("Exec %s" % cmds) shell = kwargs.get('shell', False) cmdline = cmds[0] if shell else subprocess.list2cmdline(cmds) ret = os.system(cmdline) if ret != 0: raise SystemExit("Execute '%s' error" % cmdline)
def run_cmd(cmd, envv=None, timeout=30, cwd=None, rr=False, shell=False): if type(cmd) in [str, unicode] and not shell: cmd = shlex.split(cmd) if debugging: env_string = "" if envv: env_string = " ".join(["{}='{}'".format(k, v) for k, v in envv.iteritems()]) if type(cmd) == list: print("run_cmd(" + env_string + " " + subprocess32.list2cmdline(cmd) + ")") else: print("run_cmd(" + env_string + " " + cmd + ")") # Merge current environ with passed envv merged_env = os.environ.copy() if envv: for k, v in envv.items(): merged_env[k] = v p = subprocess32.Popen(cmd, cwd=cwd, env=merged_env, stdout=PIPE, stderr=PIPE, shell=shell) try: # returns tuple (stdout, stderr) output = p.communicate(timeout=timeout) if debugging: print("Run_cmd output: {}".format(repr(output[1]))) except subprocess32.TimeoutExpired: print("Killing process due to timeout expiration.") p.terminate() return (-9, ("", "timeout expired")) return (p.returncode, output)
def __init__(self, path, mode, argv, level, inline, split=False): self.argv = argv self.level = level self.inline = inline if not inline: if split: LOG.warning("The split option only works if inline is enabled") self.fileobj = io.open(os.path.splitext(path)[0], mode) self.filehandle = self.fileobj.fileno() else: if level: if "gpg" in argv[0]: argv += ["-z%d" % level] else: argv += ["-%d" % level] self.stderr = TemporaryFile() LOG.debug("* Executing: %s", subprocess.list2cmdline(argv)) if split and not DISABLE_SPLIT: split_args = [ which("split"), "-a5", "--bytes=1G", "-", path + "." ] LOG.debug("* Splitting dump file with: %s", subprocess.list2cmdline(split_args)) self.pid = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.stderr) self.split = subprocess.Popen(split_args, stdin=self.pid.stdout, stderr=self.stderr) else: if DISABLE_SPLIT: LOG.info( "Split option is not supported with this version of subprocess module" ) self.fileobj = io.open(path, "w") self.stderr = TemporaryFile() self.pid = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=self.fileobj.fileno(), stderr=self.stderr) self.filehandle = self.pid.stdin.fileno() self.name = path self.closed = False
def install(self, filepath): ''' TODO(ssx): not tested. ''' if not os.path.exists(filepath): raise EnvironmentError('file "%s" not exists.' % filepath) ideviceinstaller = must_look_exec('ideviceinstaller') os.system( subprocess.list2cmdline( [ideviceinstaller, '-u', self.udid, '-i', filepath]))
def create_recording(qemu_path, qcow, snapshot, command, copy_directory, recording_path, expect_prompt, cdrom, isoname=None, rr=False, savevm=False, perf=False, env={}, extra_args=None): assert not (rr and perf) recording_path = realpath(recording_path) if not isoname: isoname = copy_directory + '.iso' with TempDir() as tempdir, \ Qemu(qemu_path, qcow, snapshot, tempdir, rr=rr, perf=perf, expect_prompt=expect_prompt, extra_args=extra_args) as qemu: if os.listdir(copy_directory): progress("Creating ISO {}...".format(isoname)) make_iso(copy_directory, isoname) progress("Inserting CD...") qemu.run_monitor("change {} \"{}\"".format(cdrom, isoname)) qemu.run_console("mkdir -p {}".format(pipes.quote(copy_directory))) # Make sure cdrom didn't automount # Make sure guest path mirrors host path qemu.run_console("while ! mount /dev/cdrom {}; ".format( pipes.quote(copy_directory)) + "do sleep 0.3; umount /dev/cdrom; done") # if there is a setup.sh script in the replay/proc_name/cdrom/ folder # then run that setup.sh script first (good for scriptst that need to # prep guest environment before script runs qemu.run_console("{}/setup.sh &> /dev/null || true".format( pipes.quote(copy_directory))) # Important that we type command into console before recording starts and only # hit enter once we've started the recording. progress("Running command inside guest.") qemu.type_console( subprocess32.list2cmdline(env_to_list(env) + command)) # start PANDA recording qemu.run_monitor("begin_record \"{}\"".format(recording_path)) qemu.run_console(timeout=1200) # end PANDA recording progress("Ending recording...") qemu.run_monitor("end_record")
def __init__(self, path, mode, argv, level, inline, split=False): self.argv = argv self.level = level self.inline = inline if not inline: if split: LOG.warning("The split option only works if inline is enabled") self.fileobj = io.open(os.path.splitext(path)[0], mode) self.filehandle = self.fileobj.fileno() else: if level: if "gpg" in argv[0]: argv += ["-z%d" % level] else: argv += ["-%d" % level] self.stderr = TemporaryFile() LOG.debug("* Executing: %s", subprocess.list2cmdline(argv)) if split and not DISABLE_SPLIT: split_args = [which("split"), "-a5", "--bytes=1G", "-", path + "."] LOG.debug("* Splitting dump file with: %s", subprocess.list2cmdline(split_args)) self.pid = subprocess.Popen( argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.stderr ) self.split = subprocess.Popen(split_args, stdin=self.pid.stdout, stderr=self.stderr) else: if DISABLE_SPLIT: LOG.info("Split option is not supported with this version of subprocess module") self.fileobj = io.open(path, "w") self.stderr = TemporaryFile() self.pid = subprocess.Popen( argv, stdin=subprocess.PIPE, stdout=self.fileobj.fileno(), stderr=self.stderr ) self.filehandle = self.pid.stdin.fileno() self.name = path self.closed = False
def run_cmd(cmd, envv=None, timeout=30, cwd=None, rr=False, shell=False): if type(cmd) in [str, unicode] and not shell: cmd = shlex.split(cmd) env_string = "" if envv: env_string = " ".join(["{}='{}'".format(k, v) for k, v in envv.iteritems()]) if debugging: print("run_cmd(" + env_string + " " + subprocess32.list2cmdline(cmd) + ")") p = subprocess32.Popen(cmd, cwd=cwd, env=envv, stdout=PIPE, stderr=PIPE, shell=shell) try: output = p.communicate(timeout) # returns tuple (stdout, stderr) except subprocess32.TimeoutExpired: print("Killing process due to timeout expiration.") p.terminate() return (-9, "timeout expired") return (p.returncode, output)
def time_cmd(cmd, cwd=None, env=None, timeout=None): """Calculates and outputs the time a command takes. Args: cmd (list): Command to be run. cwd (str): Working directory command is to be executed in. env (dict): Working environment command is to be executed in. timeout (int): Timeout for the command. """ if not env: env = os.environ.copy() logger.info("\nRunning `%s` now..\n", " ".join(cmd)) cmd_start = time.time() cmd = subprocess.run(cmd, cwd=cwd, env=env, timeout=timeout) cmd_end = time.time() logger.info("\n`%s` took %.3f seconds.\n", subprocess.list2cmdline(cmd.args), cmd_end - cmd_start)
def create_recording(qemu_path, qcow, snapshot, command, copy_directory, recording_path, isoname=None, rr=False): DEVNULL = open(os.devnull, "w") recording_path = realpath(recording_path) if not isoname: isoname = copy_directory + '.iso' with TempDir() as tempdir, Qemu(qemu_path, qcow, snapshot, tempdir, rr=rr) as qemu: if os.listdir(copy_directory): progress("Creating ISO {}...".format(isoname)) make_iso(copy_directory, isoname) progress("Inserting CD...") qemu.run_monitor("change ide1-cd0 \"{}\"".format(isoname)) qemu.run_console("mkdir -p {}".format(pipes.quote(copy_directory))) # Make sure cdrom didn't automount # Make sure guest path mirrors host path qemu.run_console("while ! mount /dev/cdrom {}; ".format( pipes.quote(copy_directory)) + "do sleep 0.3; umount /dev/cdrom; done") # Important that we type command into console before recording starts and only # hit enter once we've started the recording. progress("Running command inside guest.") qemu.type_console(subprocess32.list2cmdline(command)) # start PANDA recording qemu.run_monitor("begin_record \"{}\"".format(recording_path)) qemu.run_console(timeout=1200) # end PANDA recording progress("Ending recording...") qemu.run_monitor("end_record") DEVNULL.close()
def create_recording(qemu_path, qcow, snapshot, command, copy_directory, recording_path, expect_prompt, cdrom, isoname=None, rr=False, savevm=False, perf=False, env={}, extra_args=None): assert not (rr and perf) recording_path = realpath(recording_path) if not isoname: isoname = copy_directory + '.iso' with TempDir() as tempdir, \ Qemu(qemu_path, qcow, snapshot, tempdir, rr=rr, perf=perf, expect_prompt=expect_prompt, extra_args=extra_args) as qemu: if os.listdir(copy_directory): progress("Creating ISO {}...".format(isoname)) make_iso(copy_directory, isoname) progress("Inserting CD...") qemu.run_monitor("change {} \"{}\"".format(cdrom, isoname)) qemu.run_console("mkdir -p {}".format(pipes.quote(copy_directory))) # Make sure cdrom didn't automount # Make sure guest path mirrors host path qemu.run_console("while ! mount /dev/cdrom {}; ".format(pipes.quote(copy_directory)) + "do sleep 0.3; umount /dev/cdrom; done") # if there is a setup.sh script in the replay/proc_name/cdrom/ folder # then run that setup.sh script first (good for scriptst that need to # prep guest environment before script runs qemu.run_console("{}/setup.sh &> /dev/null || true".format(pipes.quote(copy_directory))) # Important that we type command into console before recording starts and only # hit enter once we've started the recording. progress("Running command inside guest.") qemu.type_console(subprocess32.list2cmdline(env_to_list(env) + command)) # start PANDA recording qemu.run_monitor("begin_record \"{}\"".format(recording_path)) qemu.run_console(timeout=1200) # end PANDA recording progress("Ending recording...") qemu.run_monitor("end_record")
qemu_args = [ project['qemu'], '-replay', isoname, '-pandalog', pandalog, '-os', panda_os_string ] for plugin, plugin_args in panda_args.iteritems(): qemu_args.append('-panda') arg_string = ",".join(["{}={}".format(arg, val) for arg, val in plugin_args.iteritems()]) qemu_args.append('{}{}{}'.format(plugin, ':' if arg_string else '', arg_string)) # Use -panda-plugin-arg to account for commas and colons in filename. qemu_args.extend(['-panda-arg', 'file_taint:filename=' + input_file_guest]) dprint("qemu args: [{}]".format(subprocess32.list2cmdline(qemu_args))) sys.stdout.flush() try: subprocess32.check_call(qemu_args, stderr=subprocess32.STDOUT) except subprocess32.CalledProcessError: if qemu_use_rr: qemu_args = ['rr', 'record', project['qemu'], '-replay', isoname] subprocess32.check_call(qemu_args) else: raise replay_time = tock() print("taint analysis complete %.2f seconds" % replay_time) sys.stdout.flush() tick()
def list2cmdline(): # list2cmdline: 把一个命令行序列转换成为命令行字符串,目前没有发现用处 result = subprocess32.list2cmdline(['-a', 'ba', '-c', 'd']) print result
def run(args, **kwargs): print("run(", subprocess32.list2cmdline(args), ")") check_call(args, cwd=lp.bugs_build, **kwargs)
def Win32Popen(cmd): """ Spawns a new process on Win32. cmd is a list of parameters. This method's sole purpose is calling CreateProcessW, not CreateProcessA as it is done by subprocess.Popen. """ import ctypes # Declare common data types DWORD = ctypes.c_uint WORD = ctypes.c_ushort LPTSTR = ctypes.c_wchar_p LPBYTE = ctypes.POINTER(ctypes.c_ubyte) HANDLE = ctypes.c_void_p class StartupInfo(ctypes.Structure): _fields_ = [("cb", DWORD), ("lpReserved", LPTSTR), ("lpDesktop", LPTSTR), ("lpTitle", LPTSTR), ("dwX", DWORD), ("dwY", DWORD), ("dwXSize", DWORD), ("dwYSize", DWORD), ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), ("dwFillAttribute", DWORD), ("dwFlags", DWORD), ("wShowWindow", WORD), ("cbReserved2", WORD), ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), ("hStdOutput", HANDLE), ("hStdError", HANDLE)] class ProcessInformation(ctypes.Structure): _fields_ = [("hProcess", HANDLE), ("hThread", HANDLE), ("dwProcessId", DWORD), ("dwThreadId", DWORD)] LPSTRARTUPINFO = ctypes.POINTER(StartupInfo) LPROCESS_INFORMATION = ctypes.POINTER(ProcessInformation) ctypes.windll.kernel32.CreateProcessW.argtypes = [LPTSTR, LPTSTR, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_bool, DWORD, ctypes.c_void_p, LPTSTR, LPSTRARTUPINFO, LPROCESS_INFORMATION] ctypes.windll.kernel32.CreateProcessW.restype = ctypes.c_bool # Convert list of arguments into a single string cmdline = subprocess.list2cmdline(cmd) buffer = ctypes.create_unicode_buffer(cmdline) # Some required structures for the method call... startupinfo = StartupInfo() ctypes.memset(ctypes.addressof(startupinfo), 0, ctypes.sizeof(startupinfo)) startupinfo.cb = ctypes.sizeof(startupinfo) processinfo = ProcessInformation() # Spawn new process success = ctypes.windll.kernel32.CreateProcessW(cmd[0], buffer, None, None, False, 0, None, None, ctypes.byref(startupinfo), ctypes.byref(processinfo)) if success: ctypes.windll.kernel32.CloseHandle(processinfo.hProcess) ctypes.windll.kernel32.CloseHandle(processinfo.hThread) return processinfo.dwProcessId else: raise ctypes.WinError(ctypes.GetLastError(), i18n.to_unicode(ctypes.FormatError()))