def test_run_cmd_async(self):
"""Test asynchronously running of a shell command via run_cmd + complete_cmd."""
os.environ['TEST'] = 'test123'
cmd_info = run_cmd("sleep 2; echo $TEST", asynchronous=True)
proc = cmd_info[0]
# change value of $TEST to check that command is completed with correct environment
os.environ['TEST'] = 'some_other_value'
# initial poll should result in None, since it takes a while for the command to complete
ec = proc.poll()
self.assertEqual(ec, None)
while ec is None:
time.sleep(1)
ec = proc.poll()
out, ec = complete_cmd(*cmd_info, simple=False)
self.assertEqual(ec, 0)
self.assertEqual(out, 'test123\n')
# also test with a command that produces a lot of output,
# since that tends to lock up things unless we frequently grab some output...
cmd = "echo start; for i in $(seq 1 50); do sleep 0.1; for j in $(seq 1000); do echo foo; done; done; echo done"
cmd_info = run_cmd(cmd, asynchronous=True)
proc = cmd_info[0]
output = ''
ec = proc.poll()
self.assertEqual(ec, None)
while ec is None:
time.sleep(1)
output += get_output_from_process(proc)
ec = proc.poll()
out, ec = complete_cmd(*cmd_info, simple=False, output=output)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith('start\n'))
self.assertTrue(out.endswith('\ndone\n'))
def test_get_output_from_process(self):
"""Test for get_output_from_process utility function."""
def get_proc(cmd, asynchronous=False):
if asynchronous:
proc = asyncprocess.Popen(cmd,
shell=True,
stdout=asyncprocess.PIPE,
stderr=asyncprocess.STDOUT,
stdin=asyncprocess.PIPE,
close_fds=True,
executable='/bin/bash')
else:
proc = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
executable='/bin/bash')
return proc
# get all output at once
proc = get_proc("echo hello")
out = get_output_from_process(proc)
self.assertEqual(out, 'hello\n')
# first get 100 bytes, then get the rest all at once
proc = get_proc("echo hello")
out = get_output_from_process(proc, read_size=100)
self.assertEqual(out, 'hello\n')
out = get_output_from_process(proc)
self.assertEqual(out, '')
# get output in small bits, keep trying to get output (which shouldn't fail)
proc = get_proc("echo hello")
out = get_output_from_process(proc, read_size=1)
self.assertEqual(out, 'h')
out = get_output_from_process(proc, read_size=3)
self.assertEqual(out, 'ell')
out = get_output_from_process(proc, read_size=2)
self.assertEqual(out, 'o\n')
out = get_output_from_process(proc, read_size=1)
self.assertEqual(out, '')
out = get_output_from_process(proc, read_size=10)
self.assertEqual(out, '')
out = get_output_from_process(proc)
self.assertEqual(out, '')
# can also get output asynchronously (read_size is *ignored* in that case)
async_cmd = "echo hello; read reply; echo $reply"
proc = get_proc(async_cmd, asynchronous=True)
out = get_output_from_process(proc, asynchronous=True)
self.assertEqual(out, 'hello\n')
asyncprocess.send_all(proc, 'test123\n')
out = get_output_from_process(proc)
self.assertEqual(out, 'test123\n')
proc = get_proc(async_cmd, asynchronous=True)
out = get_output_from_process(proc, asynchronous=True, read_size=1)
# read_size is ignored when getting output asynchronously, we're getting more than 1 byte!
self.assertEqual(out, 'hello\n')
asyncprocess.send_all(proc, 'test123\n')
out = get_output_from_process(proc, read_size=3)
self.assertEqual(out, 'tes')
out = get_output_from_process(proc, read_size=2)
self.assertEqual(out, 't1')
out = get_output_from_process(proc)
self.assertEqual(out, '23\n')
def test_run_cmd_async(self):
"""Test asynchronously running of a shell command via run_cmd + complete_cmd."""
os.environ['TEST'] = 'test123'
test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST"
cmd_info = run_cmd(test_cmd, asynchronous=True)
proc = cmd_info[0]
# change value of $TEST to check that command is completed with correct environment
os.environ['TEST'] = 'some_other_value'
# initial poll should result in None, since it takes a while for the command to complete
ec = proc.poll()
self.assertEqual(ec, None)
# wait until command is done
while ec is None:
time.sleep(1)
ec = proc.poll()
out, ec = complete_cmd(*cmd_info, simple=False)
self.assertEqual(ec, 0)
self.assertEqual(out, 'sleeping...\ntest123\n')
# also test use of check_async_cmd function
os.environ['TEST'] = 'test123'
cmd_info = run_cmd(test_cmd, asynchronous=True)
# first check, only read first 12 output characters
# (otherwise we'll be waiting until command is completed)
res = check_async_cmd(*cmd_info, output_read_size=12)
self.assertEqual(res, {
'done': False,
'exit_code': None,
'output': 'sleeping...\n'
})
# 2nd check with default output size (1024) gets full output
# (keep checking until command is fully done)
while not res['done']:
res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res, {
'done': True,
'exit_code': 0,
'output': 'sleeping...\ntest123\n'
})
# check asynchronous running of failing command
error_test_cmd = "echo 'FAIL!' >&2; exit 123"
cmd_info = run_cmd(error_test_cmd, asynchronous=True)
time.sleep(1)
error_pattern = 'cmd ".*" exited with exit code 123'
self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd,
*cmd_info)
cmd_info = run_cmd(error_test_cmd, asynchronous=True)
res = check_async_cmd(*cmd_info, fail_on_error=False)
# keep checking until command is fully done
while not res['done']:
res = check_async_cmd(*cmd_info,
fail_on_error=False,
output=res['output'])
self.assertEqual(res, {
'done': True,
'exit_code': 123,
'output': "FAIL!\n"
})
# also test with a command that produces a lot of output,
# since that tends to lock up things unless we frequently grab some output...
verbose_test_cmd = ';'.join([
"echo start",
"for i in $(seq 1 50)",
"do sleep 0.1",
"for j in $(seq 1000)",
"do echo foo",
"done",
"done",
"echo done",
])
cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
proc = cmd_info[0]
output = ''
ec = proc.poll()
self.assertEqual(ec, None)
while ec is None:
time.sleep(1)
output += get_output_from_process(proc)
ec = proc.poll()
out, ec = complete_cmd(*cmd_info, simple=False, output=output)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith('start\n'))
self.assertTrue(out.endswith('\ndone\n'))
# also test use of check_async_cmd on verbose test command
cmd_info = run_cmd(verbose_test_cmd, asynchronous=True)
error_pattern = r"Number of output bytes to read should be a positive integer value \(or zero\)"
self.assertErrorRegex(EasyBuildError,
error_pattern,
check_async_cmd,
*cmd_info,
output_read_size=-1)
self.assertErrorRegex(EasyBuildError,
error_pattern,
check_async_cmd,
*cmd_info,
output_read_size='foo')
# with output_read_size set to 0, no output is read yet, only status of command is checked
res = check_async_cmd(*cmd_info, output_read_size=0)
self.assertEqual(res['done'], False)
self.assertEqual(res['exit_code'], None)
self.assertEqual(res['output'], '')
res = check_async_cmd(*cmd_info)
self.assertEqual(res['done'], False)
self.assertEqual(res['exit_code'], None)
self.assertTrue(res['output'].startswith('start\n'))
self.assertFalse(res['output'].endswith('\ndone\n'))
# keep checking until command is complete
while not res['done']:
res = check_async_cmd(*cmd_info, output=res['output'])
self.assertEqual(res['done'], True)
self.assertEqual(res['exit_code'], 0)
self.assertTrue(res['output'].startswith('start\n'))
self.assertTrue(res['output'].endswith('\ndone\n'))
