def _reading_set_return_back(cmd, num_procs, do_return_back, compare_method):
"""
A part of test_reading_set_return_back().
Parameters
----------
cmd: str
A command to execute.
num_procs: int
A number of processes to check.
do_return_back: bool
Do return back a process after it has been ended?
compare_reversed: list
A list to compare the result.
"""
read_set = ReadingSet()
for i in range(num_procs):
proc = ReadingProc(cmd % (i + 1))
read_set.add(proc)
read_set.start_all()
returned_procs = OrderedDict()
# check returning and starting the process
i = 0
for proc, data in read_set.iter_run():
if isinstance(data.exception, ProcEnd):
proc.start()
if do_return_back:
read_set.return_back(proc)
if proc not in returned_procs:
returned_procs[proc] = 0
returned_procs[proc] += 1
i += 1
if len(returned_procs) >= num_procs:
break
assert compare_method(num_procs, list(returned_procs.values())[::-1])
def _check_pid(pid):
cmd = 'ps ax | awk \'{print $1}\' | xargs'
proc = ReadingProc(cmd)
proc.start()
stdout = ''
for data in proc.iter_run():
stdout += data.stdout.decode()
return '{}'.format(pid) in stdout
def _run_script(name, stdin_terminal=False, shell=True, env=None):
path = _get_test_script_path(name)
proc = ReadingProc( '{}'.format(path), \
stdin_terminal=stdin_terminal,
shell=shell,
env=env)
proc.start()
return proc
def test_work_dir():
"""
Testing cwd option.
"""
proc = ReadingProc('cat echo_input.py',
cwd=os.path.join(_CUR_PATH, 'test_scripts'))
proc.start()
output_lines = []
error_lines = []
for data in proc.iter_run():
output_lines.append(data.stdout.decode())
error_lines.append(data.stderr.decode())
assert all([l == '' for l in error_lines])
output = '\n'.join(output_lines)
assert '__main__' in output
def test_reading_set_operations():
"""
Check the operations: |, -, &.
Check exceptions when trying to operate with arguments of wrong types.
"""
# check exception when trying to add objects of wrong types
r1 = ReadingSet()
with pytest.raises(WrongReadingSetItem):
r1 |= set([2])
with pytest.raises(WrongReadingSetItem):
r = set([2]) | r1
with pytest.raises(WrongReadingSetItem):
r1 -= set([2])
with pytest.raises(WrongReadingSetItem):
r = set([2]) - r1
with pytest.raises(WrongReadingSetItem):
r1 = r1 & set([2])
with pytest.raises(WrongReadingSetItem):
r1 = set([2]) & r1
proc1 = ReadingProc('echo ok')
proc2 = ReadingProc('echo ok')
proc3 = ReadingProc('echo ok')
r1 |= set([proc1, proc2, proc3])
r2 = ReadingSet([proc2, proc3])
assert r2 & r1 == r1 & r2
assert r2 & r1 == r2
assert r2 & r1 == ReadingSet([proc2, proc3])
assert r1 & r2 == set([proc2, proc3])
assert r1 | r2 == r2 | r1
assert r1 | r2 == r1
r1 | r2 == ReadingSet([proc2, proc1, proc3])
r1 | r2 == set([proc1, proc2, proc3])
assert r1 - r2 == ReadingSet([proc1])
assert r1 - r2 == set([proc1])
assert r2 - r1 == ReadingSet([])
assert r2 - r1 == set([])
assert set([proc1, proc2]) == ReadingSet([proc2, proc1])
assert ReadingSet([proc2, proc1]) == set([proc1, proc2])
assert ReadingSet([proc2, proc1]) == ReadingSet([proc1, proc2])
def timeout_proc():
proc = ReadingProc('sleep 10 && echo "success"')
proc.start()
return proc
def test_common():
proc = ReadingProc(['ls', _CUR_PATH], shell=False)
proc.start()
lines = _get_lines(proc)
_check_fragment(lines, 'readingproc')
_check_fragment(lines, 'test.py')
prev_time = time()
proc = ReadingProc('sleep 3')
proc.start()
lines = _get_lines(proc)
assert time() - prev_time >= 3.0
assert len(lines) == 0
hello_world = 'Hello_world!'
proc = ReadingProc('echo {}'.format(hello_world))
proc.start()
lines = _get_lines(proc, True)
_check_fragment(lines, hello_world)
def test_reading_set_alives():
"""
Check correct work of get_all(), get_dead() and get_alive().
Check correct support of operations |=, -=
"""
cmd = 'watch -n 1 echo OK'
num_procs = 50
read_set = ReadingSet()
for i in range(num_procs):
proc = ReadingProc(cmd)
read_set |= ReadingSet([proc])
_assert_alives(read_set, num_procs, 0, num_procs)
read_set.start_all()
_assert_alives(read_set, num_procs, num_procs, 0)
read_set.terminate_all()
_assert_alives(read_set, num_procs, 0, num_procs)
# now start each process before calling start_all()
read_set = ReadingSet()
for i in range(num_procs):
proc = ReadingProc(cmd)
if i % 2 == 0:
proc.start()
read_set |= ReadingSet([proc])
else:
read_set |= ReadingSet([proc])
proc.start()
_assert_alives(read_set, num_procs, num_procs, 0)
read_set.start_all()
_assert_alives(read_set, num_procs, num_procs, 0)
all_procs = list(read_set.get_all())
all_procs[0].kill()
_assert_alives(read_set, num_procs, num_procs - 1, 1)
alive_procs = list(read_set.get_alive())
alive_procs[-1].kill()
_assert_alives(read_set, num_procs, num_procs - 2, 2)
for proc in alive_procs[0:-1]:
proc.kill()
_assert_alives(read_set, num_procs, 0, num_procs)
read_set.kill_all()
_assert_alives(read_set, num_procs, 0, num_procs)
# check terminate_all()
read_set = ReadingSet()
for i in range(num_procs):
proc = ReadingProc(cmd)
read_set.add(proc)
read_set.start_all()
_assert_alives(read_set, num_procs, num_procs, 0)
read_set.terminate_all()
_assert_alives(read_set, num_procs, 0, num_procs)
def test_reading_set_metrics(cmd_read, num_procs, total_timeout, chunk_timeout,
total_timeout_num, chunk_timeout_num,
stdout_lambda, stderr_lambda):
"""
A basic test of ReadingSet.
Parameters
----------
cmd_read: str
A reading command.
num_procs: int
A number of writing/reading procs (e.g. num_procs for each of the categories).
total_timeout: float
Using total_timeout as argument for ReadingSet.iter_run().
chunk_timeout: float
Using chunk_timeout as argument for ReadingSet.iter_run().
total_timeout_num: int
A number of total_timeouts for every patient (this is the same number for each patient). Can be 0 or 1. Stop the process.
chunk_timeout_num: int
A number of chunk_timeouts for every patient (this is the same number for each patient). Can be 0 or 1. Stop the process.
stdout_lambda: function
A function to check content of stdout.
stderr_lambda: function
A function to check content of stderr.
"""
# prepare the data
patient_indices = range(num_procs)
path = os.path.join(gettempdir(), 'patient_%(index)d')
cmd_write = 'watch -n 1 echo "patient_data%(index)d" >>{path}'.format(
path=path)
cmd_read = cmd_read.format(path=path)
write_procs = []
read_procs = {}
total_timeouts = defaultdict(lambda: 0)
chunk_timeouts = defaultdict(lambda: 0)
proc_ends = defaultdict(lambda: 0)
stdouts = defaultdict(lambda: b'')
stderrs = defaultdict(lambda: b'')
# initialize procs
for patient_index in patient_indices:
write_procs.append(ReadingProc(cmd_write % dict(index=patient_index)))
patient_path = path % dict(index=patient_index)
read_procs.update(
{ReadingProc(cmd_read % dict(index=patient_index)): patient_path})
# start checking
write_set = ReadingSet()
for i, write_proc in enumerate(write_procs):
write_set.add(write_proc)
read_set = ReadingSet()
for read_proc in read_procs:
read_set.add(read_proc)
# start all the processes
write_set.start_all()
read_set.start_all()
# check all/dead/alive before starting iterating all process outputs
_assert_alives(write_set, num_procs, num_procs, 0)
# iterating gathering statistics on read_set
i = 0
for proc, data in read_set.iter_run(total_timeout=total_timeout,
chunk_timeout=chunk_timeout):
if data.exception != None:
if isinstance(data.exception, TotalTimeout):
total_timeouts[proc] += 1
alive_before_kill = len(read_set.get_alive())
_terminate_or_kill(proc, i)
assert alive_before_kill - len(read_set.get_alive()) == 1
elif isinstance(data.exception, ChunkTimeout):
chunk_timeouts[proc] += 1
alive_before_kill = len(read_set.get_alive())
_terminate_or_kill(proc, i)
assert alive_before_kill - len(read_set.get_alive()) == 1
elif isinstance(data.exception, ProcEnd):
proc_ends[proc] += 1
else:
stdouts[proc] += data.stdout
stderrs[proc] += data.stderr
i += 1
# check statistics
assert all([x == total_timeout_num for x in total_timeouts.values()])
assert all([x == chunk_timeout_num for x in chunk_timeouts.values()])
assert all([x == 1 for x in proc_ends.values()])
assert stdout_lambda(num_procs, stdouts.values())
assert stderr_lambda(num_procs, stderrs.values())
assert all([x == 1 for x in proc_ends.values()])
# clean the data
write_set.kill_all()
read_set.terminate_all()
# check alive after stop_all()
_assert_alives(read_set, num_procs, 0, num_procs)
_assert_alives(write_set, num_procs, 0, num_procs)
# cleanup after the test,
# remove patient files
for patient_index in patient_indices:
os.remove(path % dict(index=patient_index))
def test_readingproc_destructor():
proc = ReadingProc('unknown_command')
proc.start()
for data in proc.iter_run():
pass
del proc
def _test_return_code(cmd):
proc = ReadingProc(cmd)
proc.start()
assert proc.return_code is None
sleep(4)
return proc
def test_send_cat_stdin():
cmd = 'cat'
proc = ReadingProc(cmd)
proc.start()
_test_send_stdin(proc)