def main():
"""Execution of the SCOOP module. Parses its command-line arguments and
launch needed resources."""
# Generate a argparse parser and parse the command-line arguments
parser = makeParser()
args = parser.parse_args()
# Get a list of resources to launch worker(s) on
hosts = utils.getHosts(args.hostfile, args.hosts)
if args.n:
n = args.n
else:
n = utils.getWorkerQte(hosts)
assert n >= 0, (
"Scoop couldn't determine the number of worker to start.\n"
"Use the '-n' flag to set it manually."
)
if not args.external_hostname:
args.external_hostname = [utils.externalHostname(hosts)]
# Launch SCOOP
thisScoopApp = ScoopApp(hosts, n, args.b,
args.verbose if not args.quiet else 0,
args.python_interpreter,
args.external_hostname[0],
args.executable, args.args, args.tunnel,
args.path, args.debug, args.nice,
utils.getEnv(), args.profile, args.pythonpath[0],
args.prolog[0], args.backend, args.rsh,
args.ssh_executable)
rootTaskExitCode = False
interruptPreventer = Thread(target=thisScoopApp.close)
try:
rootTaskExitCode = thisScoopApp.run()
except Exception as e:
logging.error('Error while launching SCOOP subprocesses:')
logging.error(traceback.format_exc())
rootTaskExitCode = -1
finally:
# This should not be interrupted (ie. by a KeyboadInterrupt)
# The only cross-platform way to do it I found was by using a thread.
interruptPreventer.start()
interruptPreventer.join()
# Exit with the proper exit code
if rootTaskExitCode:
sys.exit(rootTaskExitCode)
def main():
"""Execution of the SCOOP module. Parses its command-line arguments and
launch needed resources."""
# Generate a argparse parser and parse the command-line arguments
parser = makeParser()
args = parser.parse_args()
# Get a list of resources to launch worker(s) on
hosts = utils.getHosts(args.hostfile, args.hosts)
if args.n:
n = args.n
else:
n = utils.getWorkerQte(hosts)
assert n >= 0, (
"Scoop couldn't determine the number of worker to start.\n"
"Use the '-n' flag to set it manually."
)
if not args.external_hostname:
args.external_hostname = [utils.externalHostname(hosts)]
# Launch SCOOP
thisScoopApp = ScoopApp(hosts, n, args.b,
args.verbose if not args.quiet else 0,
args.python_interpreter,
args.external_hostname[0],
args.executable, args.args, args.tunnel,
args.path, args.debug, args.nice,
utils.getEnv(), args.profile, args.pythonpath[0],
args.prolog[0], args.backend, args.rsh)
rootTaskExitCode = False
interruptPreventer = Thread(target=thisScoopApp.close)
try:
rootTaskExitCode = thisScoopApp.run()
except Exception as e:
logging.error('Error while launching SCOOP subprocesses:')
logging.error(traceback.format_exc())
rootTaskExitCode = -1
finally:
# This should not be interrupted (ie. by a KeyboadInterrupt)
# The only cross-platform way to do it I found was by using a thread.
interruptPreventer.start()
interruptPreventer.join()
# Exit with the proper exit code
if rootTaskExitCode:
sys.exit(rootTaskExitCode)
def parallel(hosts_list, num_of_workers):
# Get a list of resources to launch worker(s) on
hosts = utils.getHosts(None, hosts_list)
external_hostname = [utils.externalHostname(hosts)]
# Launch SCOOP
print(sys.executable)
thisScoopApp = ScoopApp(
hosts=hosts,
n=num_of_workers,
b=1,
verbose=4,
python_executable=[sys.executable],
externalHostname=external_hostname[0],
executable="toRun.py",
arguments=None,
tunnel=False,
path="/home/martin/PycharmProjects/fineGrained/main/",
debug=False,
nice=None,
env=utils.getEnv(),
profile=None,
pythonPath=None,
prolog=None,
backend='ZMQ')
rootTaskExitCode = False
interruptPreventer = threading.Thread(target=thisScoopApp.close)
try:
rootTaskExitCode = thisScoopApp.run()
except Exception as e:
logger.error('Error while launching SCOOP subprocesses:' + str(e))
rootTaskExitCode = -1
finally:
# This should not be interrupted (ie. by a KeyboadInterrupt)
# The only cross-platform way to do it I found was by using a thread.
interruptPreventer.start()
interruptPreventer.join()
# Exit with the proper exit code
if rootTaskExitCode:
sys.exit(rootTaskExitCode)
def launch(hosts_list, num_of_workers, path, executable):
# Get a list of resources to launch worker(s) on
hosts = utils.getHosts(None, hosts_list)
external_hostname = [utils.externalHostname(hosts)]
# Launch SCOOP
app = ScoopApp(hosts=hosts,
n=num_of_workers,
b=1,
verbose=4,
python_executable=[sys.executable],
externalHostname=external_hostname[0],
executable=executable,
arguments=None,
tunnel=False,
path=path,
debug=False,
nice=None,
env=utils.getEnv(),
profile=None,
pythonPath=None,
prolog=None,
backend='ZMQ')
interrupt_prevent = threading.Thread(target=app.close)
try:
root_task_exit_code = app.run()
except Exception as e:
logger.error('Error while launching SCOOP subprocesses:' + str(e))
root_task_exit_code = -1
finally:
# This should not be interrupted (ie. by a KeyboadInterrupt)
# The only cross-platform way to do it I found was by using a thread.
interrupt_prevent.start()
interrupt_prevent.join()
# Exit with the proper exit code
print("exit code " + str(root_task_exit_code))
if root_task_exit_code:
sys.exit(root_task_exit_code)
def test_getWorkerQtePBS(self):
os.environ.update(PBS_ENV)
self.assertEqual(utils.getWorkerQte(utils.getHosts()), 16)
def test_getHostsFile(self):
self.assertEqual(set(utils.getHosts("hostfilesim.txt")), set(hosts))
def test_getHostsSGE(self):
os.environ.update(SGE_ENV)
# We used the set because the order of the hosts is not important.
self.assertEqual(set(utils.getHosts()), set(hosts))
def test_getWorkerQteFile(self):
self.assertEqual(utils.getWorkerQte(utils.getHosts("hostfilesim.txt")), 16)
def test_getHostsSGE(self):
os.environ.update(SGE_ENV)
# We used the set because the order of the hosts is not important.
self.assertEqual(set(utils.getHosts()), set(hosts))
def test_getWorkerQteFile(self):
self.assertEqual(utils.getWorkerQte(utils.getHosts("hostfilesim.txt")),
16)
def test_getWorkerQteSGE(self):
os.environ.update(SGE_ENV)
self.assertEqual(utils.getWorkerQte(utils.getHosts()), 16)
def test_getHostsFile(self):
self.assertEqual(set(utils.getHosts("hostfilesim.txt")), set(hosts))