def consume(command): """ Example of how to consume standard output and standard error of a subprocess asynchronously without risk on deadlocking. """ # Launch the command as subprocess. process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Launch the asynchronous readers of the process' stdout and stderr. stdout = AsynchronousFileReader(process.stdout, autostart=True) stderr = AsynchronousFileReader(process.stderr, autostart=True) # Check the readers if we received some output (until there is nothing more to get). while not stdout.eof() or not stderr.eof(): # Show what we received from standard output. for line in stdout.readlines(): print('Received line on standard output: ' + repr(line)) # Show what we received from standard error. for line in stderr.readlines(): print('Received line on standard error: ' + repr(line)) # Sleep a bit before polling the readers again. time.sleep(.1) # Let's be tidy and join the threads we've started. stdout.join() stderr.join() # Close subprocess' file descriptors. process.stdout.close() process.stderr.close()
def test_simple(self): file = StringIO('line1\nline2\n') reader = AsynchronousFileReader(file) lines = [] while not reader.eof(): for line in reader.readlines(): lines.append(line) reader.join() self.assertEqual(['line1\n', 'line2\n'], lines)
def test_simple(self): file = StringIO('line1\nline2\n') reader = AsynchronousFileReader(file) lines = [] while not reader.eof(): for line in reader.readlines(): lines.append(line) reader.join() self.assertEqual(['line1\n', 'line2\n'], lines)
class ProcessManager(object): def __init__(self): self._command = None self._process = None self._stdout = None self._stderr = None def start_process(self, command): self._command = command print("start process '%s'" % self._command) try: self._process = subprocess.Popen(self._command.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE) self._stdout = AsynchronousFileReader(self._process.stdout, autostart=True) #self._stderr = AsynchronousFileReader(self._process.stderr, autostart=True) except Exception: print("startup of '%s' failed" % self._command) def stop_process(self): print("stop process '%s'" % self._command) self._stdout.join() #self._stderr.join() self._process.stdout.close() #self._process.stderr.close() def get_stdout(self): lines = [] while self.is_running(): try: while not self._stdout.eof(): for line in self._stdout.readlines(): lines.append(line) yield lines lines = [] time.sleep(.1) except KeyboardInterrupt: self.stop_process() raise KeyboardInterrupt() def is_running(self): return self._process.poll() is None
def run_snap_command(command, timeout=60 * 45): """ Run a snap command. Internal use. :param command: the list of arguments to pass to snap :return: None """ # if we need to prepend the snap executable. if command[0] != os.environ['SNAP_GPT']: full_command = [os.environ['SNAP_GPT']] + command else: full_command = command # on linux there is a warning message printed by snap if this environment variable is not set. base_env = os.environ.copy() if "LD_LIBRARY_PATH" not in base_env and platform.system() != "Windows": base_env["LD_LIBRARY_PATH"] = "." logging.debug(f"running {full_command}") process = subprocess.Popen(full_command, env=base_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) process.timeout = timeout snap_logger_out = logging.getLogger("snap_stdout") snap_logger_err = logging.getLogger("snap_stderr") std_out_reader = AsynchronousFileReader(process.stdout) std_err_reader = AsynchronousFileReader(process.stderr) def pass_logging(): while not std_out_reader.queue.empty(): line = std_out_reader.queue.get().decode() snap_logger_out.info(line.rstrip('\n')) while not std_err_reader.queue.empty(): line = std_err_reader.queue.get().decode() snap_logger_err.info("stderr:" + line.rstrip('\n')) try: while process.poll() is None: pass_logging() std_out_reader.join() std_err_reader.join() except subprocess.TimeoutExpired as e: logging.error(f"IGNORING subprocess timeout running {command}") return if process.returncode != 0: raise Exception("Snap returned non zero exit status")
def _run_snap_command(command): """ Run a snap command. Internal use. :param command: the list of arguments to pass to snap :return: None """ # if we need to prepend the snap executable full_command = [r"C:\Program Files\snap\bin\gpt.exe"] + command ''' if command[0] != snap_utils.gpt_path(): full_command = [str(snap_utils.gpt_path())] + command else: full_command = command ''' # on linux there is a warning message printed by snap if this environment variable is not set. base_env = os.environ.copy() if "LD_LIBRARY_PATH" not in base_env and platform.system() != "Windows": base_env["LD_LIBRARY_PATH"] = "." logging.debug(f"running {full_command}") process = subprocess.Popen(full_command, env=base_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) snap_logger_out = logging.getLogger("snap_stdout") snap_logger_err = logging.getLogger("snap_stderr") std_out_reader = AsynchronousFileReader(process.stdout) std_err_reader = AsynchronousFileReader(process.stderr) def pass_logging(): while not std_out_reader.queue.empty(): line = std_out_reader.queue.get().decode() snap_logger_out.info(line.rstrip('\n')) while not std_err_reader.queue.empty(): line = std_err_reader.queue.get().decode() snap_logger_err.info("stderr:" + line.rstrip('\n')) while process.poll() is None: pass_logging() std_out_reader.join() std_err_reader.join() if process.returncode != 0: raise ProcessError("Snap returned non zero exit status")
def start_nfc_poll_producer(queue_to_put): logging.info('nfc poll producer started') while True: # repeatedly read output from NFC reader and put in queue process = subprocess.Popen(['unbuffer', 'nfc-poll'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = AsynchronousFileReader(process.stdout, autostart=True) stderr = AsynchronousFileReader(process.stderr, autostart=True) while not stdout.eof() or not stderr.eof(): for line in stdout.readlines(): queue_to_put.put(line.decode('utf-8')) for line in stderr.readlines(): queue_to_put.put(line.decode('utf-8')) sys_time.sleep(.1) stdout.join() stderr.join() process.stdout.close() process.stderr.close() sys_time.sleep(.1)
def runCommand(command, currentWorkingDirectory, callerArguments=None, init_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): if __builtin__.type(expectedExitCodes) is not list: raise TypeError("{}({}) is not {}".format( "expectedExitCodes", __builtin__.type(expectedExitCodes), list)) if __builtin__.type(onlyErrorCaseOutput) is not bool: raise TypeError("{}({}) is not {}".format( "onlyErrorCaseOutput", __builtin__.type(onlyErrorCaseOutput), bool)) if __builtin__.type(command) is list: commandAsList = command else: commandAsList = command[:].split(' ') environment = getEnvironment(init_environment, callerArguments) # add some necessary paths if hasattr(callerArguments, 'gitpath' ) and callerArguments.gitpath and commandAsList[0] == 'git': commandAsList[0] = os.path.abspath( os.path.join(callerArguments.gitpath, 'git')) if hasattr(callerArguments, 'perlpath' ) and callerArguments.perlpath and commandAsList[0] == 'perl': commandAsList[0] = os.path.abspath( os.path.join(callerArguments.perlpath, 'perl')) if hasattr(callerArguments, 'sevenzippath' ) and callerArguments.sevenzippath and commandAsList[0] == '7z': commandAsList[0] = os.path.abspath( os.path.join(callerArguments.sevenzippath, '7z')) # if we can not find the command, just check the current working dir if not os.path.lexists(commandAsList[0]) and currentWorkingDirectory and \ os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))): commandAsList[0] = os.path.abspath( os.path.join(currentWorkingDirectory, commandAsList[0])) if 'Path' in environment: pathEnvironment = environment['Path'] elif 'PATH' in environment: pathEnvironment = environment['PATH'] # if we can not find the command, check the environment if not os.path.lexists(commandAsList[0]) and find_executable( commandAsList[0], pathEnvironment): commandAsList[0] = find_executable(commandAsList[0], pathEnvironment) if currentWorkingDirectory and not os.path.lexists( currentWorkingDirectory): os.makedirs(currentWorkingDirectory) print(os.linesep + '========================== do ... ==========================') if currentWorkingDirectory: print("Working Directory: " + currentWorkingDirectory) else: print("No currentWorkingDirectory set!") print("Last command: " + ' '.join(commandAsList)) if currentWorkingDirectory and not os.path.lexists( currentWorkingDirectory): raise Exception("The current working directory is not existing: %s" % currentWorkingDirectory) useShell = True if sys.platform.startswith('win') else False lastStdOutLines = [] lastStdErrLines = [] if threading.currentThread( ).name == "MainThread" and not onlyErrorCaseOutput: process = subprocess.Popen(commandAsList, shell=useShell, cwd=currentWorkingDirectory, bufsize=-1, env=environment) else: process = subprocess.Popen(commandAsList, shell=useShell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=currentWorkingDirectory, bufsize=-1, env=environment) maxSavedLineNumbers = 100 lastStdOutLines = collections.deque(maxlen=maxSavedLineNumbers) lastStdErrLines = collections.deque(maxlen=maxSavedLineNumbers) # Launch the asynchronous readers of the process' stdout and stderr. stdout = AsynchronousFileReader(process.stdout) stderr = AsynchronousFileReader(process.stderr) # Check the readers if we received some output (until there is nothing more to get). while not stdout.eof() or not stderr.eof(): # Show what we received from standard output. for line in stdout.readlines(): lastStdOutLines.append(line) if threading.currentThread().name != "MainThread": sys.stdout.write(line) # Show what we received from standard error. for line in stderr.readlines(): lastStdErrLines.append(line) if threading.currentThread().name != "MainThread": sys.stdout.write(line) # Sleep a bit before polling the readers again. time.sleep(1) # Let's be tidy and join the threads we've started. stdout.join() stderr.join() # Close subprocess' file descriptors. process.stdout.close() process.stderr.close() process.wait() exitCode = process.returncode # lets keep that for debugging #if environment: # for key in sorted(environment): # sys.stderr.write("set " + key + "=" + environment[key] + os.linesep) if exitCode not in expectedExitCodes: lastOutput = "" type = "" if threading.currentThread( ).name != "MainThread" or onlyErrorCaseOutput: if len(lastStdErrLines) != 0: lastOutput += "".join(lastStdErrLines) type = "error " elif len(lastStdOutLines) != 0: lastOutput += "".join(lastStdOutLines) prettyLastOutput = os.linesep + '======================= error =======================' + os.linesep prettyLastOutput += "Working Directory: " + currentWorkingDirectory + os.linesep prettyLastOutput += "Last command: " + ' '.join( commandAsList) + os.linesep if lastOutput: prettyLastOutput += "last {0}output:{1}{2}".format( type, os.linesep, lastOutput) else: prettyLastOutput += " - no process output caught - " raise Exception( "Different exit code then expected({0}): {1}{2}".format( expectedExitCodes, exitCode, prettyLastOutput)) return exitCode
def runCommand(command, currentWorkingDirectory, callerArguments = None, init_environment = None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): if __builtin__.type(expectedExitCodes) is not list: raise TypeError("{}({}) is not {}".format("expectedExitCodes", __builtin__.type(expectedExitCodes), list)) if __builtin__.type(onlyErrorCaseOutput) is not bool: raise TypeError("{}({}) is not {}".format("onlyErrorCaseOutput", __builtin__.type(onlyErrorCaseOutput), bool)) if __builtin__.type(command) is list: commandAsList = command else: commandAsList = command[:].split(' ') environment = getEnvironment(init_environment, callerArguments) # add some necessary paths if hasattr(callerArguments, 'gitpath') and callerArguments.gitpath and commandAsList[0] == 'git': commandAsList[0] = os.path.abspath(os.path.join(callerArguments.gitpath, 'git')) if hasattr(callerArguments, 'perlpath') and callerArguments.perlpath and commandAsList[0] == 'perl': commandAsList[0] = os.path.abspath(os.path.join(callerArguments.perlpath, 'perl')) if hasattr(callerArguments, 'sevenzippath') and callerArguments.sevenzippath and commandAsList[0] == '7z': commandAsList[0] = os.path.abspath(os.path.join(callerArguments.sevenzippath, '7z')) # if we can not find the command, just check the current working dir if not os.path.lexists(commandAsList[0]) and currentWorkingDirectory and \ os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))): commandAsList[0] = os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0])) if 'Path' in environment: pathEnvironment = environment['Path'] elif 'PATH' in environment: pathEnvironment = environment['PATH'] # if we can not find the command, check the environment if not os.path.lexists(commandAsList[0]) and find_executable(commandAsList[0], pathEnvironment): commandAsList[0] = find_executable(commandAsList[0], pathEnvironment) if currentWorkingDirectory and not os.path.lexists(currentWorkingDirectory): os.makedirs(currentWorkingDirectory) print(os.linesep + '========================== do ... ==========================') if currentWorkingDirectory: print("Working Directory: " + currentWorkingDirectory) else: print("No currentWorkingDirectory set!") print("Last command: " + ' '.join(commandAsList)) if currentWorkingDirectory and not os.path.lexists(currentWorkingDirectory): raise Exception("The current working directory is not existing: %s" % currentWorkingDirectory) useShell = True if sys.platform.startswith('win') else False lastStdOutLines = [] lastStdErrLines = [] if threading.currentThread().name == "MainThread" and not onlyErrorCaseOutput: process = subprocess.Popen(commandAsList, shell=useShell, cwd = currentWorkingDirectory, bufsize = -1, env = environment) else: process = subprocess.Popen(commandAsList, shell=useShell, stdout = subprocess.PIPE, stderr = subprocess.PIPE, cwd = currentWorkingDirectory, bufsize = -1, env = environment) maxSavedLineNumbers = 100 lastStdOutLines = collections.deque(maxlen = maxSavedLineNumbers) lastStdErrLines = collections.deque(maxlen = maxSavedLineNumbers) # Launch the asynchronous readers of the process' stdout and stderr. stdout = AsynchronousFileReader(process.stdout) stderr = AsynchronousFileReader(process.stderr) # Check the readers if we received some output (until there is nothing more to get). while not stdout.eof() or not stderr.eof(): # Show what we received from standard output. for line in stdout.readlines(): lastStdOutLines.append(line) if threading.currentThread().name != "MainThread": sys.stdout.write(line) # Show what we received from standard error. for line in stderr.readlines(): lastStdErrLines.append(line) if threading.currentThread().name != "MainThread": sys.stdout.write(line) # Sleep a bit before polling the readers again. time.sleep(1) # Let's be tidy and join the threads we've started. stdout.join() stderr.join() # Close subprocess' file descriptors. process.stdout.close() process.stderr.close() process.wait() exitCode = process.returncode # lets keep that for debugging #if environment: # for key in sorted(environment): # sys.stderr.write("set " + key + "=" + environment[key] + os.linesep) if exitCode not in expectedExitCodes: lastOutput = "" type = "" if threading.currentThread().name != "MainThread" or onlyErrorCaseOutput: if len(lastStdErrLines) != 0: lastOutput += "".join(lastStdErrLines) type = "error " elif len(lastStdOutLines) != 0: lastOutput += "".join(lastStdOutLines) prettyLastOutput = os.linesep + '======================= error =======================' + os.linesep prettyLastOutput += "Working Directory: " + currentWorkingDirectory + os.linesep prettyLastOutput += "Last command: " + ' '.join(commandAsList) + os.linesep if lastOutput: prettyLastOutput += "last {0}output:{1}{2}".format(type, os.linesep, lastOutput) else: prettyLastOutput += " - no process output caught - " raise Exception("Different exit code then expected({0}): {1}{2}".format(expectedExitCodes, exitCode, prettyLastOutput)) return exitCode
# Check the queues if we received some output (until there is nothing more to get). while not stdout_reader.eof() or not stderr_reader.eof(): # Show what we received from standard output. while not stdout_queue.empty(): line = stdout_queue.get() process_job_output(db, cur, details['JOBID'], line) if DEBUG: print line, # Show what we received from standard error. while not stderr_queue.empty(): line = stderr_queue.get() process_job_output(db, cur, details['JOBID'], line) if DEBUG: print line, # Sleep a bit before asking the readers again. time.sleep(.1) # Let's be tidy and join the threads we've started. stdout_reader.join() stderr_reader.join() # Close subprocess' file descriptors. process.stdout.close() process.stderr.close() # Clean up db.close()