def test_simple(self):
file = StringIO('line1\nline2\n')
reader = AsynchronousFileReader(file)
lines = []
while not reader.eof():
for line in reader.readlines():
lines.append(line)
reader.join()
self.assertEqual(['line1\n', 'line2\n'], lines)
class ProcessManager(object):
def __init__(self):
self._command = None
self._process = None
self._stdout = None
self._stderr = None
def start_process(self, command):
self._command = command
print("start process '%s'" % self._command)
try:
self._process = subprocess.Popen(self._command.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self._stdout = AsynchronousFileReader(self._process.stdout,
autostart=True)
#self._stderr = AsynchronousFileReader(self._process.stderr, autostart=True)
except Exception:
print("startup of '%s' failed" % self._command)
def stop_process(self):
print("stop process '%s'" % self._command)
self._stdout.join()
#self._stderr.join()
self._process.stdout.close()
#self._process.stderr.close()
def get_stdout(self):
lines = []
while self.is_running():
try:
while not self._stdout.eof():
for line in self._stdout.readlines():
lines.append(line)
yield lines
lines = []
time.sleep(.1)
except KeyboardInterrupt:
self.stop_process()
raise KeyboardInterrupt()
def is_running(self):
return self._process.poll() is None
def start_process(self, command):
self._command = command
print("start process '%s'" % self._command)
try:
self._process = subprocess.Popen(self._command.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self._stdout = AsynchronousFileReader(self._process.stdout,
autostart=True)
#self._stderr = AsynchronousFileReader(self._process.stderr, autostart=True)
except Exception:
print("startup of '%s' failed" % self._command)
def test_with(self):
data_stream = StringIO('line1\nline2\n')
with AsynchronousFileReader(data_stream) as reader:
lines = []
while not reader.eof():
for line in reader.readlines():
lines.append(line)
self.assertEqual(['line1\n', 'line2\n'], lines)
def run_snap_command(command, timeout=60 * 45):
"""
Run a snap command. Internal use.
:param command: the list of arguments to pass to snap
:return: None
"""
# if we need to prepend the snap executable.
if command[0] != os.environ['SNAP_GPT']:
full_command = [os.environ['SNAP_GPT']] + command
else:
full_command = command
# on linux there is a warning message printed by snap if this environment variable is not set.
base_env = os.environ.copy()
if "LD_LIBRARY_PATH" not in base_env and platform.system() != "Windows":
base_env["LD_LIBRARY_PATH"] = "."
logging.debug(f"running {full_command}")
process = subprocess.Popen(full_command,
env=base_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process.timeout = timeout
snap_logger_out = logging.getLogger("snap_stdout")
snap_logger_err = logging.getLogger("snap_stderr")
std_out_reader = AsynchronousFileReader(process.stdout)
std_err_reader = AsynchronousFileReader(process.stderr)
def pass_logging():
while not std_out_reader.queue.empty():
line = std_out_reader.queue.get().decode()
snap_logger_out.info(line.rstrip('\n'))
while not std_err_reader.queue.empty():
line = std_err_reader.queue.get().decode()
snap_logger_err.info("stderr:" + line.rstrip('\n'))
try:
while process.poll() is None:
pass_logging()
std_out_reader.join()
std_err_reader.join()
except subprocess.TimeoutExpired as e:
logging.error(f"IGNORING subprocess timeout running {command}")
return
if process.returncode != 0:
raise Exception("Snap returned non zero exit status")
def _run_snap_command(command):
"""
Run a snap command. Internal use.
:param command: the list of arguments to pass to snap
:return: None
"""
# if we need to prepend the snap executable
full_command = [r"C:\Program Files\snap\bin\gpt.exe"] + command
'''
if command[0] != snap_utils.gpt_path():
full_command = [str(snap_utils.gpt_path())] + command
else:
full_command = command
'''
# on linux there is a warning message printed by snap if this environment variable is not set.
base_env = os.environ.copy()
if "LD_LIBRARY_PATH" not in base_env and platform.system() != "Windows":
base_env["LD_LIBRARY_PATH"] = "."
logging.debug(f"running {full_command}")
process = subprocess.Popen(full_command,
env=base_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
snap_logger_out = logging.getLogger("snap_stdout")
snap_logger_err = logging.getLogger("snap_stderr")
std_out_reader = AsynchronousFileReader(process.stdout)
std_err_reader = AsynchronousFileReader(process.stderr)
def pass_logging():
while not std_out_reader.queue.empty():
line = std_out_reader.queue.get().decode()
snap_logger_out.info(line.rstrip('\n'))
while not std_err_reader.queue.empty():
line = std_err_reader.queue.get().decode()
snap_logger_err.info("stderr:" + line.rstrip('\n'))
while process.poll() is None:
pass_logging()
std_out_reader.join()
std_err_reader.join()
if process.returncode != 0:
raise ProcessError("Snap returned non zero exit status")
def test_simple(self):
file = StringIO('line1\nline2\n')
reader = AsynchronousFileReader(file)
lines = []
while not reader.eof():
for line in reader.readlines():
lines.append(line)
reader.join()
self.assertEqual(['line1\n', 'line2\n'], lines)
def runCommand(command,
currentWorkingDirectory,
callerArguments=None,
init_environment=None,
onlyErrorCaseOutput=False,
expectedExitCodes=[0]):
if __builtin__.type(expectedExitCodes) is not list:
raise TypeError("{}({}) is not {}".format(
"expectedExitCodes", __builtin__.type(expectedExitCodes), list))
if __builtin__.type(onlyErrorCaseOutput) is not bool:
raise TypeError("{}({}) is not {}".format(
"onlyErrorCaseOutput", __builtin__.type(onlyErrorCaseOutput),
bool))
if __builtin__.type(command) is list:
commandAsList = command
else:
commandAsList = command[:].split(' ')
environment = getEnvironment(init_environment, callerArguments)
# add some necessary paths
if hasattr(callerArguments, 'gitpath'
) and callerArguments.gitpath and commandAsList[0] == 'git':
commandAsList[0] = os.path.abspath(
os.path.join(callerArguments.gitpath, 'git'))
if hasattr(callerArguments, 'perlpath'
) and callerArguments.perlpath and commandAsList[0] == 'perl':
commandAsList[0] = os.path.abspath(
os.path.join(callerArguments.perlpath, 'perl'))
if hasattr(callerArguments, 'sevenzippath'
) and callerArguments.sevenzippath and commandAsList[0] == '7z':
commandAsList[0] = os.path.abspath(
os.path.join(callerArguments.sevenzippath, '7z'))
# if we can not find the command, just check the current working dir
if not os.path.lexists(commandAsList[0]) and currentWorkingDirectory and \
os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))):
commandAsList[0] = os.path.abspath(
os.path.join(currentWorkingDirectory, commandAsList[0]))
if 'Path' in environment:
pathEnvironment = environment['Path']
elif 'PATH' in environment:
pathEnvironment = environment['PATH']
# if we can not find the command, check the environment
if not os.path.lexists(commandAsList[0]) and find_executable(
commandAsList[0], pathEnvironment):
commandAsList[0] = find_executable(commandAsList[0], pathEnvironment)
if currentWorkingDirectory and not os.path.lexists(
currentWorkingDirectory):
os.makedirs(currentWorkingDirectory)
print(os.linesep +
'========================== do ... ==========================')
if currentWorkingDirectory:
print("Working Directory: " + currentWorkingDirectory)
else:
print("No currentWorkingDirectory set!")
print("Last command: " + ' '.join(commandAsList))
if currentWorkingDirectory and not os.path.lexists(
currentWorkingDirectory):
raise Exception("The current working directory is not existing: %s" %
currentWorkingDirectory)
useShell = True if sys.platform.startswith('win') else False
lastStdOutLines = []
lastStdErrLines = []
if threading.currentThread(
).name == "MainThread" and not onlyErrorCaseOutput:
process = subprocess.Popen(commandAsList,
shell=useShell,
cwd=currentWorkingDirectory,
bufsize=-1,
env=environment)
else:
process = subprocess.Popen(commandAsList,
shell=useShell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=currentWorkingDirectory,
bufsize=-1,
env=environment)
maxSavedLineNumbers = 100
lastStdOutLines = collections.deque(maxlen=maxSavedLineNumbers)
lastStdErrLines = collections.deque(maxlen=maxSavedLineNumbers)
# Launch the asynchronous readers of the process' stdout and stderr.
stdout = AsynchronousFileReader(process.stdout)
stderr = AsynchronousFileReader(process.stderr)
# Check the readers if we received some output (until there is nothing more to get).
while not stdout.eof() or not stderr.eof():
# Show what we received from standard output.
for line in stdout.readlines():
lastStdOutLines.append(line)
if threading.currentThread().name != "MainThread":
sys.stdout.write(line)
# Show what we received from standard error.
for line in stderr.readlines():
lastStdErrLines.append(line)
if threading.currentThread().name != "MainThread":
sys.stdout.write(line)
# Sleep a bit before polling the readers again.
time.sleep(1)
# Let's be tidy and join the threads we've started.
stdout.join()
stderr.join()
# Close subprocess' file descriptors.
process.stdout.close()
process.stderr.close()
process.wait()
exitCode = process.returncode
# lets keep that for debugging
#if environment:
# for key in sorted(environment):
# sys.stderr.write("set " + key + "=" + environment[key] + os.linesep)
if exitCode not in expectedExitCodes:
lastOutput = ""
type = ""
if threading.currentThread(
).name != "MainThread" or onlyErrorCaseOutput:
if len(lastStdErrLines) != 0:
lastOutput += "".join(lastStdErrLines)
type = "error "
elif len(lastStdOutLines) != 0:
lastOutput += "".join(lastStdOutLines)
prettyLastOutput = os.linesep + '======================= error =======================' + os.linesep
prettyLastOutput += "Working Directory: " + currentWorkingDirectory + os.linesep
prettyLastOutput += "Last command: " + ' '.join(
commandAsList) + os.linesep
if lastOutput:
prettyLastOutput += "last {0}output:{1}{2}".format(
type, os.linesep, lastOutput)
else:
prettyLastOutput += " - no process output caught - "
raise Exception(
"Different exit code then expected({0}): {1}{2}".format(
expectedExitCodes, exitCode, prettyLastOutput))
return exitCode
def start_nfc_poll_producer(queue_to_put):
logging.info('nfc poll producer started')
while True:
# repeatedly read output from NFC reader and put in queue
process = subprocess.Popen(['unbuffer', 'nfc-poll'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout = AsynchronousFileReader(process.stdout, autostart=True)
stderr = AsynchronousFileReader(process.stderr, autostart=True)
while not stdout.eof() or not stderr.eof():
for line in stdout.readlines():
queue_to_put.put(line.decode('utf-8'))
for line in stderr.readlines():
queue_to_put.put(line.decode('utf-8'))
sys_time.sleep(.1)
stdout.join()
stderr.join()
process.stdout.close()
process.stderr.close()
sys_time.sleep(.1)
def runCommand(command, currentWorkingDirectory, callerArguments = None, init_environment = None, onlyErrorCaseOutput=False, expectedExitCodes=[0]):
if __builtin__.type(expectedExitCodes) is not list:
raise TypeError("{}({}) is not {}".format("expectedExitCodes", __builtin__.type(expectedExitCodes), list))
if __builtin__.type(onlyErrorCaseOutput) is not bool:
raise TypeError("{}({}) is not {}".format("onlyErrorCaseOutput", __builtin__.type(onlyErrorCaseOutput), bool))
if __builtin__.type(command) is list:
commandAsList = command
else:
commandAsList = command[:].split(' ')
environment = getEnvironment(init_environment, callerArguments)
# add some necessary paths
if hasattr(callerArguments, 'gitpath') and callerArguments.gitpath and commandAsList[0] == 'git':
commandAsList[0] = os.path.abspath(os.path.join(callerArguments.gitpath, 'git'))
if hasattr(callerArguments, 'perlpath') and callerArguments.perlpath and commandAsList[0] == 'perl':
commandAsList[0] = os.path.abspath(os.path.join(callerArguments.perlpath, 'perl'))
if hasattr(callerArguments, 'sevenzippath') and callerArguments.sevenzippath and commandAsList[0] == '7z':
commandAsList[0] = os.path.abspath(os.path.join(callerArguments.sevenzippath, '7z'))
# if we can not find the command, just check the current working dir
if not os.path.lexists(commandAsList[0]) and currentWorkingDirectory and \
os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))):
commandAsList[0] = os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))
if 'Path' in environment:
pathEnvironment = environment['Path']
elif 'PATH' in environment:
pathEnvironment = environment['PATH']
# if we can not find the command, check the environment
if not os.path.lexists(commandAsList[0]) and find_executable(commandAsList[0], pathEnvironment):
commandAsList[0] = find_executable(commandAsList[0], pathEnvironment)
if currentWorkingDirectory and not os.path.lexists(currentWorkingDirectory):
os.makedirs(currentWorkingDirectory)
print(os.linesep + '========================== do ... ==========================')
if currentWorkingDirectory:
print("Working Directory: " + currentWorkingDirectory)
else:
print("No currentWorkingDirectory set!")
print("Last command: " + ' '.join(commandAsList))
if currentWorkingDirectory and not os.path.lexists(currentWorkingDirectory):
raise Exception("The current working directory is not existing: %s" % currentWorkingDirectory)
useShell = True if sys.platform.startswith('win') else False
lastStdOutLines = []
lastStdErrLines = []
if threading.currentThread().name == "MainThread" and not onlyErrorCaseOutput:
process = subprocess.Popen(commandAsList, shell=useShell,
cwd = currentWorkingDirectory, bufsize = -1, env = environment)
else:
process = subprocess.Popen(commandAsList, shell=useShell,
stdout = subprocess.PIPE, stderr = subprocess.PIPE,
cwd = currentWorkingDirectory, bufsize = -1, env = environment)
maxSavedLineNumbers = 100
lastStdOutLines = collections.deque(maxlen = maxSavedLineNumbers)
lastStdErrLines = collections.deque(maxlen = maxSavedLineNumbers)
# Launch the asynchronous readers of the process' stdout and stderr.
stdout = AsynchronousFileReader(process.stdout)
stderr = AsynchronousFileReader(process.stderr)
# Check the readers if we received some output (until there is nothing more to get).
while not stdout.eof() or not stderr.eof():
# Show what we received from standard output.
for line in stdout.readlines():
lastStdOutLines.append(line)
if threading.currentThread().name != "MainThread":
sys.stdout.write(line)
# Show what we received from standard error.
for line in stderr.readlines():
lastStdErrLines.append(line)
if threading.currentThread().name != "MainThread":
sys.stdout.write(line)
# Sleep a bit before polling the readers again.
time.sleep(1)
# Let's be tidy and join the threads we've started.
stdout.join()
stderr.join()
# Close subprocess' file descriptors.
process.stdout.close()
process.stderr.close()
process.wait()
exitCode = process.returncode
# lets keep that for debugging
#if environment:
# for key in sorted(environment):
# sys.stderr.write("set " + key + "=" + environment[key] + os.linesep)
if exitCode not in expectedExitCodes:
lastOutput = ""
type = ""
if threading.currentThread().name != "MainThread" or onlyErrorCaseOutput:
if len(lastStdErrLines) != 0:
lastOutput += "".join(lastStdErrLines)
type = "error "
elif len(lastStdOutLines) != 0:
lastOutput += "".join(lastStdOutLines)
prettyLastOutput = os.linesep + '======================= error =======================' + os.linesep
prettyLastOutput += "Working Directory: " + currentWorkingDirectory + os.linesep
prettyLastOutput += "Last command: " + ' '.join(commandAsList) + os.linesep
if lastOutput:
prettyLastOutput += "last {0}output:{1}{2}".format(type, os.linesep, lastOutput)
else:
prettyLastOutput += " - no process output caught - "
raise Exception("Different exit code then expected({0}): {1}{2}".format(expectedExitCodes, exitCode, prettyLastOutput))
return exitCode
def consume(command):
"""
Example of how to consume standard output and standard error of
a subprocess asynchronously without risk on deadlocking.
"""
# Launch the command as subprocess.
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Launch the asynchronous readers of the process' stdout and stderr.
stdout = AsynchronousFileReader(process.stdout, autostart=True)
stderr = AsynchronousFileReader(process.stderr, autostart=True)
# Check the readers if we received some output (until there is nothing more to get).
while not stdout.eof() or not stderr.eof():
# Show what we received from standard output.
for line in stdout.readlines():
print('Received line on standard output: ' + repr(line))
# Show what we received from standard error.
for line in stderr.readlines():
print('Received line on standard error: ' + repr(line))
# Sleep a bit before polling the readers again.
time.sleep(.1)
# Let's be tidy and join the threads we've started.
stdout.join()
stderr.join()
# Close subprocess' file descriptors.
process.stdout.close()
process.stderr.close()
-file JOBSCRIPT_DIR/egrep-dropped.sh \
-file JOBSCRIPT_DIR/noreducer.sh'''
# Replace variables with necessary details
for key, value in details.items():
startcmd = startcmd.replace(key, value)
if DEBUG:
print startcmd
# Start search job
process = subprocess.Popen(startcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# Launch the asynchronous readers of the process' stdout and stderr.
stdout_queue = Queue()
stdout_reader = AsynchronousFileReader(process.stdout, stdout_queue)
stdout_reader.start()
stderr_queue = Queue()
stderr_reader = AsynchronousFileReader(process.stderr, stderr_queue)
stderr_reader.start()
# Check the queues if we received some output (until there is nothing more to get).
while not stdout_reader.eof() or not stderr_reader.eof():
# Show what we received from standard output.
while not stdout_queue.empty():
line = stdout_queue.get()
process_job_output(db, cur, details['JOBID'], line)
if DEBUG:
print line,
# Show what we received from standard error.
