def __init__(self, nb_workers):
self._lt = LogsFollower()
self._logger = WorkflowRunner.get_logger()
self._pool = None
if nb_workers == -1:
self._nb_workers = int((cpu_count() + 1) / 2)
else:
self._nb_workers = nb_workers
self._free_workers = None
self._completed_processes = []
def test_thread_protection(self):
"""When a section of code using the LogsFollower is complete, the thread should stop"""
lf = LogsFollower()
lf.follow_process("stdout", "stderr", "process_id")
with lf.trace_in_background():
assert lf._thread.is_alive(), "Backgroung thread isn't running..."
with open("stdout", "w") as fout, \
open("stderr", "w") as ferr:
fout.write("file stdout")
ferr.write("file stderr")
assert not lf._thread.is_alive(), "Backgroung hasn't stopped !"
def test_log_format_stdout_stderr(self):
"""logs should display log level and message"""
with CaptureOutputs() as co:
lf = LogsFollower()
lf.follow_process("stdout", "stderr", "process_id")
with open("stdout", "w") as fout, \
open("stderr", "w") as ferr:
fout.write("file stdout")
ferr.write("file stderr")
while lf.trace_logs():
pass
assert co.output.find(
"[process_id::stdout] file stdout") >= 0, co.output
assert co.output.find(
"[process_id::stderr] file stderr") >= 0, co.output
def test_log_in_background(self):
"""Should log in background ans stop when foreground processing
is over"""
import time
with CaptureOutputs() as co:
lf = LogsFollower()
lf.follow_process("stdout", "stderr", "process_id")
lf.trace_in_background()
with open("stdout", "w") as fout, \
open("stderr", "w") as ferr:
fout.write("file stdout")
ferr.write("file stderr")
lf.terminate()
assert co.output.find(
"[process_id::stdout] file stdout") >= 0, co.output
assert co.output.find(
"[process_id::stderr] file stderr") >= 0, co.output
def run_pre_processes(self):
""" Runs all the preprocesses
:return:
:raises ExecutionError if an error occurs
"""
TuttleDirectories.create_tuttle_dirs()
TuttleDirectories.empty_extension_dir()
if not self.has_preprocesses():
return
lt = LogsFollower()
WorkflowRunner.print_preprocesses_header()
for process in self.iter_preprocesses():
TuttleDirectories.prepare_and_assign_paths(process)
lt.follow_process(process.log_stdout, process.log_stderr, None)
with lt.trace_in_background(), TuttleEnv():
for preprocess in self.iter_preprocesses():
WorkflowRunner.print_preprocess_header(preprocess, lt._logger)
success = True
error_msg = None
try:
preprocess.set_start()
preprocess.processor.run(preprocess,
preprocess._reserved_path,
preprocess.log_stdout,
preprocess.log_stderr)
except TuttleError as e:
success = False
error_msg = str(e)
raise
except Exception:
exc_info = sys.exc_info()
stacktrace = "".join(format_exception(*exc_info))
error_msg = "An unexpected error have happened in tuttle preprocessor {} : \n" \
"{}\n" \
"Preprocess {} will not complete.".format(preprocess._processor.name, stacktrace, preprocess.id)
finally:
preprocess.set_end(success, error_msg)
self.create_reports()
WorkflowRunner.print_preprocesses_footer()
def test_log_multiple_files(self):
"""LogTracer should log the content of several files in stdout"""
with CaptureOutputs() as co:
lf = LogsFollower()
lf.follow_process("w1.stdout", "w1.stderr", "process1")
lf.follow_process("w2.stdout", "w2.stderr", "process2")
lf.follow_process("w3.stdout", "w3.stderr", "process3")
with open("w1.stdout", "w") as fo1, \
open("w1.stderr", "w") as fe1, \
open("w2.stdout", "w") as fo2, \
open("w2.stderr", "w") as fe2, \
open("w3.stdout", "w") as fo3, \
open("w3.stderr", "w") as fe3 :
for i in xrange(5000):
fo1.write("w1.stdout - line {}\n".format(i))
fe1.write("w1.stderr - line {}\n".format(i))
fo2.write("w2.stdout - line {}\n".format(i))
fe2.write("w2.stderr - line {}\n".format(i))
fo3.write("w3.stdout - line {}\n".format(i))
fe3.write("w3.stderr - line {}\n".format(i))
while lf.trace_logs():
pass
output = co.output
assert output.find("w1.stderr - line 1") >= 0, output
assert output.find("w1.stdout - line 1") >= 0, output
assert output.find("w2.stderr - line 1") >= 0, output
assert output.find("w2.stdout - line 1") >= 0, output
assert output.find("w3.stdout - line 1") >= 0, output
assert output.find("w3.stderr - line 1") >= 0, output
assert output.find("w1.stderr - line 4999") >= 0, output
assert output.find("w1.stdout - line 4999") >= 0, output
assert output.find("w2.stderr - line 4999") >= 0, output
assert output.find("w2.stdout - line 4999") >= 0, output
assert output.find("w3.stdout - line 4999") >= 0, output
assert output.find("w3.stderr - line 4999") >= 0, output
def test_log_a_lot_in_background(self):
"""Should log in background ans stop when foreground processing
is over even with a lot a data"""
with CaptureOutputs() as co:
lf = LogsFollower()
lf.follow_process("stdout", "stderr", "process_id")
lf.trace_in_background()
with open("stdout", "w") as fout, \
open("stderr", "w") as ferr:
fout.write("file stdout")
ferr.write("file stderr")
for i in xrange(5000):
fout.write("stdout - line {}\n".format(i))
ferr.write("stderr - line {}\n".format(i))
lf.terminate()
assert co.output.find(
"[process_id::stdout] stdout - line 1") >= 0, co.output
assert co.output.find(
"[process_id::stderr] stderr - line 1") >= 0, co.output
assert co.output.find(
"[process_id::stdout] stdout - line 4999") >= 0, co.output
assert co.output.find(
"[process_id::stderr] stderr - line 4999") >= 0, co.output