def test_parallel(self):
cmd = "python -c 'import time; time.sleep(.75)'"
cmd_list = [cmd for nn in range(4)]
then = datetime.datetime.now()
parallel_lib.parallel(
cmd_list,
njobs=1,
suppress_stdout=True,
suppress_stderr=True,
assume_hyperthread=True,
suppress_cmd=True,
)
seconds_single = (datetime.datetime.now() - then).total_seconds()
then = datetime.datetime.now()
parallel_lib.parallel(
cmd_list,
njobs=4,
suppress_stdout=True,
suppress_stderr=True,
assume_hyperthread=True,
suppress_cmd=True,
)
seconds_multi = (datetime.datetime.now() - then).total_seconds()
self.assertTrue(seconds_single / seconds_multi >= 3.)
def test_parallel(self):
cmd = "python -c 'import time; time.sleep(.75)'"
cmd_list = [cmd for nn in range(4)]
then = datetime.datetime.now()
parallel_lib.parallel(
cmd_list, njobs=1, suppress_stdout=True, suppress_stderr=True,
assume_hyperthread=True,
suppress_cmd=True,
)
seconds_single = (datetime.datetime.now() - then).total_seconds()
then = datetime.datetime.now()
parallel_lib.parallel(
cmd_list, njobs=4, suppress_stdout=True, suppress_stderr=True,
assume_hyperthread=True,
suppress_cmd=True,
)
seconds_multi = (datetime.datetime.now() - then).total_seconds()
self.assertTrue(seconds_single / seconds_multi >= 3.)
def main():
msg = "Tool to run shell commands in parallel. Spawns processes "
msg += "to concurrently run commands supplied on stdin. "
msg = textwrap.dedent(
"""
Read a list of commands from stdin and execute them in parrallel.
-----------------------------------------------------------------------
Examples:
* This line generates commands that will be used in the examples.
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader
* Execute the commands one at a time, no parallelism
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 1
* Execute all commands in parallel
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 10
* Suppress stdout from processes and echo commands
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 10 -c -s stdout
* Make a histogram of how long the individual jobs took
time seq 100 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 50 -v \\
| grep __job__ \\
| p.df 'df.dropna()' 'df.duration_sec.hist(bins=20)'
-----------------------------------------------------------------------
"""
)
# read command line arguments
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, description=msg)
msg = "The number of jobs to run in parallel. If not supplied, will "
msg += "default to the number of detected cores."
parser.add_argument('--njobs', '-n', dest='njobs', default=[None],
nargs=1, type=int, help=msg)
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="Enable verbose output")
parser.add_argument("-c", "--show_commands", action="store_true",
default=False, help="Print commands to stdout")
msg = "Suppress stdout, stderr, or both for all running jobs"
parser.add_argument("-s", "--suppress",
choices=['stdout', 'stderr', 'both'], default=[None],
nargs=1, help=msg)
# add standard arg groups
arg_lib.add_args(parser, 'example')
# parse arguments
args = parser.parse_args()
# get the commands from stdin
cmd_list = sys.stdin.readlines()
# get suppression vars from args
suppress_stdout = 'stdout' in args.suppress or 'both' in args.suppress
suppress_stderr = 'stderr' in args.suppress or 'both' in args.suppress
# run the commands
parallel_lib.parallel(
cmd_list,
njobs=args.njobs[0],
verbose=args.verbose,
suppress_cmd=(not args.show_commands),
suppress_stdout=suppress_stdout,
suppress_stderr=suppress_stderr,
assume_hyperthread=True)
def main():
msg = "Tool to run shell commands in parallel. Spawns processes "
msg += "to concurrently run commands supplied on stdin. "
msg = textwrap.dedent("""
Read a list of commands from stdin and execute them in parrallel.
-----------------------------------------------------------------------
Examples:
* This line generates commands that will be used in the examples.
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader
* Execute the commands one at a time, no parallelism
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 1
* Execute all commands in parallel
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 10
* Suppress stdout from processes and echo commands
time seq 10 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 10 -c -s stdout
* Make a histogram of how long the individual jobs took
time seq 100 \\
| p.format -t 'sleep 1; echo done {n}' --names n -i noheader \\
| p.parallel -n 50 -v \\
| grep __job__ \\
| p.df 'df.dropna()' 'df.duration_sec.hist(bins=20)'
-----------------------------------------------------------------------
""")
# read command line arguments
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, description=msg)
msg = "The number of jobs to run in parallel. If not supplied, will "
msg += "default to the number of detected cores."
parser.add_argument('--njobs',
'-n',
dest='njobs',
default=[None],
nargs=1,
type=int,
help=msg)
parser.add_argument("-v",
"--verbose",
action="store_true",
default=False,
help="Enable verbose output")
parser.add_argument("-c",
"--show_commands",
action="store_true",
default=False,
help="Print commands to stdout")
msg = "Suppress stdout, stderr, or both for all running jobs"
parser.add_argument("-s",
"--suppress",
choices=['stdout', 'stderr', 'both'],
default=[None],
nargs=1,
help=msg)
# parse arguments
args = parser.parse_args()
# get the commands from stdin
cmd_list = sys.stdin.readlines()
# get suppression vars from args
suppress_stdout = 'stdout' in args.suppress or 'both' in args.suppress
suppress_stderr = 'stderr' in args.suppress or 'both' in args.suppress
# run the commands
parallel_lib.parallel(cmd_list,
njobs=args.njobs[0],
verbose=args.verbose,
suppress_cmd=(not args.show_commands),
suppress_stdout=suppress_stdout,
suppress_stderr=suppress_stderr,
assume_hyperthread=True)
