def create_new_process(self): ''' Creating a new process ''' if 'win' in PLATFORM: self._stdout = open(self._stdout_path, 'w+b') self._process = Popen(args=self._command, stdin=PIPE, stdout=self._stdout) self._pipe = self._process.stdin else: if os.path.exists(self._pipe_path): os.unlink(self._pipe_path) os.mkfifo(self._pipe_path) self._pipe = open(self._pipe_path, 'w+b') if os.path.exists(self._stdout_path): os.remove(self._stdout_path) self._stdout = open(self._stdout_path, 'w+b') self._process = Popen(args=self._command, stdin=self._pipe, stdout=self._stdout) self._player_answer = open(self._stdout_path, 'r') self._pid = self._process.pid # Writing pid to file pid_file = open(self._pid_path, 'w') pid_file.write(str(self._pid)) pid_file.close() # Passing pipe and stdout to properties class self.properties.__init__(self._pipe, self._player_answer, self._debug)
def shell_exec(args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False): """ Wrapper for subprocess starting :param stderr: :param stdout: :param cwd: :param stdin: :type args: basestring or list :return: """ if stdout and not isinstance(stdout, int) and not isinstance(stdout, file_type): logging.warning("stdout is not IOBase: %s", stdout) stdout = None if stderr and not isinstance(stderr, int) and not isinstance(stderr, file_type): logging.warning("stderr is not IOBase: %s", stderr) stderr = None if isinstance(args, string_types) and not shell: args = shlex.split(args) logging.getLogger(__name__).debug("Executing shell: %s", args) if platform.system() == 'Windows': return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, cwd=cwd, shell=shell) else: return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, preexec_fn=os.setpgrp, close_fds=True, cwd=cwd, shell=shell)
def test_streams(self): self.sniff_log(logging.getLogger('')) print('test1') with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG): print('test2') with log_std_streams(stdout_level=logging.DEBUG): print('test3') with log_std_streams(stdout_level=logging.DEBUG): sys.stdout.write('test3') with log_std_streams(logger=self.captured_logger, stdout_level=logging.DEBUG): process = Popen(['echo', '"test5"']) process.wait() missed_file = get_uniq_name('.', 'test6', '') with log_std_streams(logger=self.captured_logger, stderr_level=logging.WARNING): process = Popen(['dir', missed_file]) process.wait() debug_buf = self.log_recorder.debug_buff.getvalue() warn_buf = self.log_recorder.warn_buff.getvalue() self.assertNotIn('test1', debug_buf) self.assertIn('test2', debug_buf) self.assertNotIn('test3', debug_buf) self.assertIn('test5', debug_buf) self.assertTrue(len(warn_buf) > 0)
def convert_to_mobi(assembled_ebooks): converted_ebooks = {} kindle_gen_exists = Popen('kindlegen', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) kindle_gen_exists.communicate() if kindle_gen_exists.returncode == 0: for volume, ebook_file in assembled_ebooks.items(): info('Converting file "{}" to MOBI...'.format(ebook_file)) kindle_gen_convert = Popen('kindlegen "{}" -c2'.format(ebook_file), stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) kindle_gen_convert.communicate() if kindle_gen_convert.returncode not in [ 0, 1 ]: # acceptable exit codes error('Error to convert file "{}" to MOBI'.format(ebook_file)) continue os.remove(ebook_file) converted_ebooks[volume] = os.path.join( ebook_file, str(ebook_file).replace('.epub', '.mobi')) else: warning('KindleGen not found!') return converted_ebooks
def checkTools(source): source = source.upper() if source.endswith('.CBR') or source.endswith('.RAR'): rarExitCode = Popen('unrar', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) rarExitCode = rarExitCode.wait() if rarExitCode != 0 and rarExitCode != 1 and rarExitCode != 7: print('ERROR: UnRAR is missing!') exit(1) elif source.endswith('.CB7') or source.endswith('.7Z'): sevenzaExitCode = Popen('7za', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) sevenzaExitCode = sevenzaExitCode.wait() if sevenzaExitCode != 0 and sevenzaExitCode != 7: print('ERROR: 7za is missing!') exit(1) if options.format == 'MOBI': kindleGenExitCode = Popen('kindlegen -locale en', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) if kindleGenExitCode.wait() != 0: print('ERROR: KindleGen is missing!') exit(1)
def shell_exec(args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE): """ Wrapper for subprocess starting :param stderr: :param stdout: :param cwd: :param stdin: :type args: basestring or list :return: """ if isinstance(args, six.string_types): args = shlex.split(args) logging.getLogger(__name__).debug("Executing shell: %s", args) if platform.system() == 'Windows': return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, cwd=cwd) else: return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, preexec_fn=os.setpgrp, close_fds=True, cwd=cwd)
def test_streams(self): self.log = logging.getLogger('') handler = RecordingHandler() self.log.addHandler(handler) print('test1') with log_std_streams(logger=self.log, stdout_level=logging.DEBUG): print('test2') with log_std_streams(stdout_level=logging.DEBUG): print('test3') with log_std_streams(stdout_level=logging.DEBUG): sys.stdout.write('test3') with log_std_streams(logger=self.log, stdout_level=logging.DEBUG): process = Popen(['echo', '"test5"']) process.wait() missed_file = get_uniq_name('.', 'test6', '') with log_std_streams(logger=self.log, stderr_level=logging.WARNING): process = Popen(['dir', missed_file]) process.wait() self.log.removeHandler(handler) debug_buf = handler.debug_buff.getvalue() warn_buf = handler.warn_buff.getvalue() self.assertNotIn('test1', debug_buf) self.assertIn('test2', debug_buf) self.assertNotIn('test3', debug_buf) self.assertIn('test5', debug_buf) self.assertTrue(len(warn_buf) > 0)
def shell_exec(args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False, env=None): """ Wrapper for subprocess starting :param stderr: :param stdout: :param cwd: :param stdin: :type args: basestring or list :return: """ if stdout and not isinstance(stdout, int) and not isinstance(stdout, file_type): logging.warning("stdout is not IOBase: %s", stdout) stdout = None if stderr and not isinstance(stderr, int) and not isinstance(stderr, file_type): logging.warning("stderr is not IOBase: %s", stderr) stderr = None if isinstance(args, string_types) and not shell: args = shlex.split(args, posix=not is_windows()) logging.getLogger(__name__).debug("Executing shell: %s", args) if env: env = {k: str(v) for k, v in iteritems(env)} if is_windows(): return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, cwd=cwd, shell=shell, env=env) else: return Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, bufsize=0, preexec_fn=os.setpgrp, close_fds=True, cwd=cwd, shell=shell, env=env)
def _run(self): # fix for some broken saved jobs for i, j in enumerate(self.jobs): if type(j) == str or type(j) == np.string_: self.jobs[i] = eval(j) errors = len([j for j in self.jobs if j['status'] == 'X']) if errors: print "Previous run had %d errors. Retrying." % errors for j in self.jobs: if j['status'] == 0 or j['status'] == 'X': cmd = j['cmd'] cpus = min(j['cpu'], self.cpus) if cpus > self._cpus_free: self.wait(cpus) self._cpus_free -= cpus sout = open(j['outfile'] + '.out', 'w') serr = open(j['outfile'] + '.err', 'w') if not windows: cmd = self.timestr + ' ' + cmd if j['dir']: cmd = 'cd %s;%s' % (j['dir'], cmd) # We are going to wait for each process, so we must keep the Popen object # around, otherwise it will quietly wait for the process and exit, # leaving our wait function waiting for nonexistent processes. p = Popen(cmd, shell=True, stdout=sout, stderr=serr) j['status'] = 'R' self._running.append((p, j)) self._monitor.start_job(j['cmd'], p.pid) self.wait(0)
def __init__(self, ros_port, ws_port, uid): self.ros_port = ros_port self.ws_port = ws_port self.uid = uid self.vis = Popen( shlex.split('python start_ros.py {} {}'.format(ros_port, ws_port)))
def extractCB7(self, targetdir): # Workaround for some wide UTF-8 + Popen abnormalities if sys.platform.startswith('darwin'): copy( self.origFileName, os.path.join(os.path.dirname(self.origFileName), 'TMP_KCC_TMP')) self.origFileName = os.path.join( os.path.dirname(self.origFileName), 'TMP_KCC_TMP') output = Popen( '7za x "' + self.origFileName + '" -xr!__MACOSX -xr!.DS_Store -xr!thumbs.db -xr!Thumbs.db -o"' + targetdir + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) extracted = False for line in output.stdout: if b"Everything is Ok" in line: extracted = True if sys.platform.startswith('darwin'): os.remove(self.origFileName) if not extracted: raise OSError
def __init__(self, source): self.source = source self.data = { 'Series': '', 'Volume': '', 'Number': '', 'Writers': [], 'Pencillers': [], 'Inkers': [], 'Colorists': [], 'Summary': '', 'MUid': '', 'Bookmarks': [] } self.rawdata = None self.compressor = None if self.source.endswith('.xml'): self.rawdata = parse(self.source) self.parseXML() else: if is_zipfile(self.source): self.compressor = 'zip' with ZipFile(self.source) as zip_file: for member in zip_file.namelist(): if member != 'ComicInfo.xml': continue with zip_file.open(member) as xml_file: self.rawdata = parse(xml_file) elif rarfile.is_rarfile(self.source): self.compressor = 'rar' with rarfile.RarFile(self.source) as rar_file: for member in rar_file.namelist(): if member != 'ComicInfo.xml': continue with rar_file.open(member) as xml_file: self.rawdata = parse(xml_file) elif is_7zfile(self.source): self.compressor = '7z' workdir = mkdtemp('', 'KCC-') tmpXML = os.path.join(workdir, 'ComicInfo.xml') output = Popen('7za e "' + self.source + '" ComicInfo.xml -o"' + workdir + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) extracted = False for line in output.stdout: if b"Everything is Ok" in line or b"No files to process" in line: extracted = True if not extracted: rmtree(workdir) raise OSError('Failed to extract 7ZIP file.') if os.path.isfile(tmpXML): self.rawdata = parse(tmpXML) rmtree(workdir) else: raise OSError('Failed to detect archive format.') if self.rawdata: self.parseXML()
def check(): data = request.json t = time() out_file = f'output{t}.txt' code_file = f'code{t}.php' output = open(out_file, 'w+') with open(code_file, 'w') as f: f.write(data['code']) process = Popen(['php', '-d', php_disable_functions, code_file], stdout=output, stderr=output) exit_code = -1 result = '' try: exit_code = process.wait(5) except Exception as e: print(e) if process.is_running(): process.kill() output.seek(0) result = output.read() output.close() os.remove(out_file) os.remove(code_file) return jsonify({'exit_code': exit_code, 'result': result})
def _run(self): """Run McStas simulation executable. The current settings of the instrument parameters will be transferred to it. """ try: shutil.rmtree(self.mcstasdir) except (IOError, OSError): self.log.info('could not remove old data') command = '%s -n 1e8 -d %s %s' % (self.mcstasprog, self.mcstasdir, self._mcstas_params) self.log.debug('run %s', command) try: self._process = Popen(command.split(), stdout=PIPE, stderr=PIPE, cwd=self._workdir) out, err = self._process.communicate() if out: self.log.debug('McStas output:') for line in out.splitlines(): self.log.debug('[McStas] %s', line) if err: self.log.warning('McStas found some problems:') for line in err.splitlines(): self.log.warning('[McStas] %s', line) except OSError as e: self.log.error('Execution failed: %s', e) self._process.wait() self._process = None
def connect(cls, data_dir, jar_file): register(RemoteServer.disconnect) jar = path.split(jar_file)[-1] rpc_host, rpc_port = "127.0.0.1", "50051" for p in process_iter(attrs=["cmdline"]): cmdline = p.info.get("cmdline") if (cmdline and cmdline[0].endswith("java")): index = next( (i for i, x in enumerate(cmdline) if x.endswith(jar)), -1) if (index > 0): rpc_port = cmdline[index + 1] break else: sock = socket() sock.bind((rpc_host, 0)) rpc_port = f"{sock.getsockname()[1]}" sock.close() java_home = environ.get("JAVA_HOME") if (java_home and path.isdir(path.join(java_home, "bin"))): java_exec = path.join(java_home, "bin", "java") else: java_exec = "java" cls.rpc_server = Popen([ java_exec, "-Xmx2G", "-XX:+UseG1GC", "-jar", jar_file, rpc_port, data_dir ]) rpc_uri = f"{rpc_host}:{rpc_port}" cls.rpc_channel = insecure_channel( rpc_uri, options=[("grpc.max_send_message_length", 2147483647), ("grpc.max_receive_message_length", 2147483647)]) channel_ready_future(cls.rpc_channel).result(timeout=30.0)
def _start_worker(self, index: int): """ Start a new worker child process """ logger.info("Creating a new worker") path = shutil.which("chaosplatform-worker") name = self.worker_name if self.add_suffix: name = "{}-{}".format(name, uuid.uuid4().hex) args = [path, "--name", name, "--queue", self.queue_name] if self.log_level == "DEBUG": args.append("--verbose") # notice, the current's process environment variables will be passed # down to the child worker process logger.debug("Executing: {}".format(" ".join(args))) p = Popen(args, stdout=None, stderr=None, shell=False, cwd=self.worker_directory) self._processes.append(p) logger.info("Worker '{}' started (PID {})".format(name, p.pid))
def start_exec(self, command, work_dir: str = None, shell: bool = False, errors=(), successes=()): self.clean() logger.debug(f"Using work dir: {work_dir}") work_path = Path(work_dir) work_path.mkdir(exist_ok=True, parents=True) self.output_file = work_path / f"encoder_output_{secrets.token_hex(6)}.log" self.error_output_file = work_path / f"encoder_error_output_{secrets.token_hex(6)}.log" logger.debug(f"command output file set to: {self.output_file}") logger.debug(f"command error output file set to: {self.error_output_file}") self.output_file.touch(exist_ok=True) self.error_output_file.touch(exist_ok=True) self.error_message = errors self.success_message = successes logger.info(f"Running command: {command}") self.process = Popen( shlex.split(command.replace("\\", "\\\\")) if not shell and isinstance(command, str) else command, shell=shell, cwd=work_dir, stdout=open(self.output_file, "w"), stderr=open(self.error_output_file, "w"), stdin=PIPE, # FFmpeg can try to read stdin and wrecks havoc on linux encoding="utf-8", ) self.started_at = datetime.datetime.now(datetime.timezone.utc) Thread(target=self.read_output).start()
def start_exec(self, command, work_dir: str = None, shell: bool = False, errors=(), successes=()): self.clean() logger.info(f"Running command: {command}") Path(work_dir).mkdir(exist_ok=True, parents=True) self.output_file = Path( work_dir) / f"encoder_output_{secrets.token_hex(6)}.log" self.error_output_file = Path( work_dir) / f"encoder_error_output_{secrets.token_hex(6)}.log" self.output_file.touch(exist_ok=True) self.error_output_file.touch(exist_ok=True) self.error_message = errors self.success_message = successes self.process = Popen( shlex.split(command) if not shell and isinstance(command, str) else command, shell=shell, cwd=work_dir, stdout=open(self.output_file, "w"), stderr=open(self.error_output_file, "w"), stdin=PIPE, # FFmpeg can try to read stdin and wrecks havoc on linux encoding="utf-8", ) Thread(target=self.read_output).start()
def reset(self): self.gazebo.unpauseSim() self.check_all_sensors_ready() self.z_collision = 0 self.depth = 0 self.command.twist.linear.x = 0 self.pub.publish(self.command) self.pub2.publish(0) self.pub3.publish(0) self.node_process.terminate() self.set_init_pose() obs = self._get_obs() now = rospy.get_rostime() self._is_done(obs, now) self.gazebo.pauseSim() self.gazebo.resetSim() self.node_process = Popen( shlex.split('rosrun robil_lihi C_response_loader.py')) self.gazebo.unpauseSim() self.command.twist.linear.x = 0 self.pub.publish(self.command) self.pub2.publish(0) self.pub3.publish(0) self.set_init_pose() self.init_env_variables() self.check_all_sensors_ready() rospy.sleep(1) self.gazebo.pauseSim() self.init_env_variables() obs = self._get_obs() simplified_obs = self.convert_obs_to_state(obs) plt.clf() return simplified_obs
def sshcmd_shell(self, dst, cmd): output = [] errors = [] ex = [ 'ssh', '-l root', '-q', '-o UserKnownHostsFile=/dev/null', '-o StrictHostKeyChecking=no', '-o ConnectTimeout=10', dst ] ex += cmd self.logger.debug("Execution list: %r", ex) execline = ' '.join(ex) self.logger.debug("Exec line: %r", execline) sp = Popen(execline, env=os.environ, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = sp.communicate() output = [l.strip() for l in out.decode('utf-8').split('\n') if len(l)] self.logger.debug("SSH CMD return code: %r", sp.returncode) return sp.returncode, output, errors
def addFile(self, sourcefile): if self.type in ['RAR', 'RAR5']: raise NotImplementedError process = Popen('7z a -y "' + self.filepath + '" "' + sourcefile + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) process.communicate() if process.returncode != 0: raise OSError('Failed to add the file.')
def execute_module(executable, package_name, env): args = [executable, "-m", package_name] with Popen(args, env=env, stdout=subprocess.PIPE) as process: process.wait() retcode = process.returncode output = process.stdout.read().decode("utf-8") return retcode, output
def spawn(self): sockets_fds = self._get_sockets_fds() args = self.format_args(sockets_fds=sockets_fds) def preexec_fn(): streams = [sys.stdin] if self.close_child_stdout: streams.append(sys.stdout) if self.close_child_stderr: streams.append(sys.stderr) self._null_streams(streams) os.setsid() for limit, value in self.rlimits.items(): res = getattr(resource, 'RLIMIT_%s' % limit.upper(), None) if res is None: raise ValueError('unknown rlimit "%s"' % limit) # TODO(petef): support hard/soft limits resource.setrlimit(res, (value, value)) if self.gid: try: os.setgid(self.gid) except OverflowError: if not ctypes: raise # versions of python < 2.6.2 don't manage unsigned int for # groups like on osx or fedora os.setgid(-ctypes.c_int(-self.gid).value) os.initgroups(self.username, self.gid) if self.uid: os.setuid(self.uid) extra = {} if self.pipe_stdout: extra['stdout'] = PIPE if self.pipe_stderr: extra['stderr'] = PIPE self._worker = Popen(args, cwd=self.working_dir, shell=self.shell, preexec_fn=preexec_fn, env=self.env, close_fds=not self.use_fds, executable=self.executable, **extra) # let go of sockets created only for self._worker to inherit self._sockets = [] self.started = time.time()
def test_get_info(self): worker = Popen(["python -c 'import time;time.sleep(5)'"], shell=True) try: info = get_info(worker) finally: worker.terminate() self.assertTrue(isinstance(info['pid'], int)) self.assertEqual(info['nice'], 0)
def extractCB7(self, targetdir): output = Popen('7za x "' + self.fname + '" -xr!__MACOSX -xr!.DS_Store -xr!thumbs.db -xr!Thumbs.db -o"' + targetdir + '"', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) extracted = False for line in output.stdout: if b"Everything is Ok" in line: extracted = True if not extracted: raise OSError
def run_subprocess( command: Sequence[str], shell: bool = False, doexec: bool = True, monitor_log: logging.Logger = None, monitor_interval: int = 5, tile_id: str = None, ) -> bool: """Runs a subprocess with `psutil.Popen` and monitors its status. If subprocess returns non-zero exit code, STDERR is sent to the log. :param command: The command to execute. :param shell: Passed to `psutil.Popen`. Defaults to False. :param doexec: Do execute the subprocess or just print out the concatenated command. Used for testing. :param monitor_log: A resource logger, which is returned by :func:`~.recorder.configure_resource_logging`. :param monitor_interval: How often query the resource usage of the process? In seconds. :param tile_id: Used for monitoring only. :return: True/False on success/failure """ if doexec: cmd = " ".join(command) if shell: command = cmd log.debug(f"Tile {tile_id} command: {command}") start = time() popen = Popen(command, shell=shell, stderr=PIPE, stdout=PIPE) if monitor_log is not None: while True: sleep(monitor_interval) monitor_log.info( f"{tile_id}\t{popen.pid}\t{popen.cpu_times().user}" f"\t{popen.memory_info().rss}") if (not popen.is_running() or popen.status() == STATUS_ZOMBIE or popen.status() == STATUS_SLEEPING): break stdout, stderr = popen.communicate() err = stderr.decode(getpreferredencoding(do_setlocale=True)) out = stdout.decode(getpreferredencoding(do_setlocale=True)) finish = time() log.info(f"Tile {tile_id} finished in {(finish-start)/60} minutes") if popen.returncode != 0: log.error( f"Tile {tile_id} process returned with {popen.returncode}") else: log.debug( f"Tile {tile_id} process returned with {popen.returncode}") log.debug(f"Tile {tile_id} stdout: \n{out}") log.debug(f"Tile {tile_id} stderr: \n{err}") return True if popen.returncode == 0 else False else: log.debug(f"Tile {tile_id} not executing {command}") return True
def extractMetadata(self): process = Popen('7z x -y -so "' + self.filepath + '" ComicInfo.xml', stdout=PIPE, stderr=STDOUT, stdin=PIPE, shell=True) xml = process.communicate() if process.returncode != 0: raise OSError('Failed to extract archive.') try: return parseString(xml[0]) except ExpatError: return None
def run_script(script, *args): s = os.path.join(os.path.dirname(__file__), '../../scripts/' + script) command = [executable, s] command.extend(args) with Popen([executable, s]) as p: sleep(4) p.send_signal(SIGINT) p.wait(timeout=1) assert p.poll() == 0, 'script failed'
def startUpButtonf(): if self.startUpChoice == False: Popen( 'copy ' + winPath + 'startup\\startup.exe "C:\\ProgramData\\Microsoft\\Windows\\Start Menu\\Programs\\StartUp"', shell=True) with open('C:/Easy USB Backup Info.pyc', mode='w') as i: i.write(winPath + "main.exe") Log("开机自启已选择") self.startUpText.set("开机自启") self.startUpChoice = True elif self.startUpChoice == True: Popen( 'del ' + '"C:\\ProgramData\\Microsoft\\Windows\\Start Menu\\Programs\\StartUp\\startup.exe"', shell=True) Log("开机不自启已选择") self.startUpText.set("开机不自启") self.startUpChoice = False
def create_memory_eating_child(self, block_size): from psutil import Popen from subprocess import PIPE return Popen([ sys.executable, '-c', 'eat_memory = "x" * %d ; ' % (block_size, ) + 'import sys ; ' + 'sys.stdout.write("x") ; ' + 'sys.stdout.flush() ; ' + 'sys.stdin.read(1)' ], stdin=PIPE, stdout=PIPE)