Exemple #1
0
def main():
    """Execution of the SCOOP module. Parses its command-line arguments and
    launch needed resources."""
    # Generate a argparse parser and parse the command-line arguments
    parser = makeParser()
    args = parser.parse_args()

    # Get a list of resources to launch worker(s) on
    hosts = utils.getHosts(args.hostfile, args.hosts)

    if args.n:
        n = args.n
    else:
        n = utils.getWorkerQte(hosts)
    assert n >= 0, (
            "Scoop couldn't determine the number of worker to start.\n"
            "Use the '-n' flag to set it manually."
    )

    if not args.external_hostname:
        args.external_hostname = [utils.externalHostname(hosts)]

    # Launch SCOOP
    thisScoopApp = ScoopApp(hosts, n, args.b,
                            args.verbose if not args.quiet else 0,
                            args.python_interpreter,
                            args.external_hostname[0],
                            args.executable, args.args, args.tunnel,
                            args.path, args.debug, args.nice,
                            utils.getEnv(), args.profile, args.pythonpath[0],
                            args.prolog[0], args.backend, args.rsh,
                            args.ssh_executable)

    rootTaskExitCode = False
    interruptPreventer = Thread(target=thisScoopApp.close)
    try:
        rootTaskExitCode = thisScoopApp.run()
    except Exception as e:
        logging.error('Error while launching SCOOP subprocesses:')
        logging.error(traceback.format_exc())
        rootTaskExitCode = -1
    finally:
        # This should not be interrupted (ie. by a KeyboadInterrupt)
        # The only cross-platform way to do it I found was by using a thread.
        interruptPreventer.start()
        interruptPreventer.join()

    # Exit with the proper exit code
    if rootTaskExitCode:
        sys.exit(rootTaskExitCode)
Exemple #2
0
def main():
    """Execution of the SCOOP module. Parses its command-line arguments and
    launch needed resources."""
    # Generate a argparse parser and parse the command-line arguments
    parser = makeParser()
    args = parser.parse_args()

    # Get a list of resources to launch worker(s) on
    hosts = utils.getHosts(args.hostfile, args.hosts)

    if args.n:
        n = args.n
    else:
        n = utils.getWorkerQte(hosts)
    assert n >= 0, (
            "Scoop couldn't determine the number of worker to start.\n"
            "Use the '-n' flag to set it manually."
    )

    if not args.external_hostname:
        args.external_hostname = [utils.externalHostname(hosts)]

    # Launch SCOOP
    thisScoopApp = ScoopApp(hosts, n, args.b,
                            args.verbose if not args.quiet else 0,
                            args.python_interpreter,
                            args.external_hostname[0],
                            args.executable, args.args, args.tunnel,
                            args.path, args.debug, args.nice,
                            utils.getEnv(), args.profile, args.pythonpath[0],
                            args.prolog[0], args.backend, args.rsh)

    rootTaskExitCode = False
    interruptPreventer = Thread(target=thisScoopApp.close)
    try:
        rootTaskExitCode = thisScoopApp.run()
    except Exception as e:
        logging.error('Error while launching SCOOP subprocesses:')
        logging.error(traceback.format_exc())
        rootTaskExitCode = -1
    finally:
        # This should not be interrupted (ie. by a KeyboadInterrupt)
        # The only cross-platform way to do it I found was by using a thread.
        interruptPreventer.start()
        interruptPreventer.join()

    # Exit with the proper exit code
    if rootTaskExitCode:
        sys.exit(rootTaskExitCode)
Exemple #3
0
def parallel(hosts_list, num_of_workers):
    # Get a list of resources to launch worker(s) on
    hosts = utils.getHosts(None, hosts_list)
    external_hostname = [utils.externalHostname(hosts)]
    # Launch SCOOP
    print(sys.executable)
    thisScoopApp = ScoopApp(
        hosts=hosts,
        n=num_of_workers,
        b=1,
        verbose=4,
        python_executable=[sys.executable],
        externalHostname=external_hostname[0],
        executable="toRun.py",
        arguments=None,
        tunnel=False,
        path="/home/martin/PycharmProjects/fineGrained/main/",
        debug=False,
        nice=None,
        env=utils.getEnv(),
        profile=None,
        pythonPath=None,
        prolog=None,
        backend='ZMQ')

    rootTaskExitCode = False
    interruptPreventer = threading.Thread(target=thisScoopApp.close)
    try:
        rootTaskExitCode = thisScoopApp.run()
    except Exception as e:
        logger.error('Error while launching SCOOP subprocesses:' + str(e))
        rootTaskExitCode = -1
    finally:
        # This should not be interrupted (ie. by a KeyboadInterrupt)
        # The only cross-platform way to do it I found was by using a thread.
        interruptPreventer.start()
        interruptPreventer.join()

    # Exit with the proper exit code
    if rootTaskExitCode:
        sys.exit(rootTaskExitCode)
Exemple #4
0
def launch(hosts_list, num_of_workers, path, executable):
    # Get a list of resources to launch worker(s) on
    hosts = utils.getHosts(None, hosts_list)
    external_hostname = [utils.externalHostname(hosts)]
    # Launch SCOOP
    app = ScoopApp(hosts=hosts,
                   n=num_of_workers,
                   b=1,
                   verbose=4,
                   python_executable=[sys.executable],
                   externalHostname=external_hostname[0],
                   executable=executable,
                   arguments=None,
                   tunnel=False,
                   path=path,
                   debug=False,
                   nice=None,
                   env=utils.getEnv(),
                   profile=None,
                   pythonPath=None,
                   prolog=None,
                   backend='ZMQ')

    interrupt_prevent = threading.Thread(target=app.close)
    try:
        root_task_exit_code = app.run()
    except Exception as e:
        logger.error('Error while launching SCOOP subprocesses:' + str(e))
        root_task_exit_code = -1
    finally:
        # This should not be interrupted (ie. by a KeyboadInterrupt)
        # The only cross-platform way to do it I found was by using a thread.
        interrupt_prevent.start()
        interrupt_prevent.join()

    # Exit with the proper exit code
    print("exit code " + str(root_task_exit_code))
    if root_task_exit_code:
        sys.exit(root_task_exit_code)
Exemple #5
0
 def test_getWorkerQtePBS(self):
     os.environ.update(PBS_ENV)
     self.assertEqual(utils.getWorkerQte(utils.getHosts()), 16)
Exemple #6
0
 def test_getHostsFile(self):
     self.assertEqual(set(utils.getHosts("hostfilesim.txt")), set(hosts))
Exemple #7
0
 def test_getHostsSGE(self):
     os.environ.update(SGE_ENV)
     # We used the set because the order of the hosts is not important.
     self.assertEqual(set(utils.getHosts()), set(hosts))
Exemple #8
0
 def test_getWorkerQteFile(self):
     self.assertEqual(utils.getWorkerQte(utils.getHosts("hostfilesim.txt")), 16)
Exemple #9
0
 def test_getHostsSGE(self):
     os.environ.update(SGE_ENV)
     # We used the set because the order of the hosts is not important.
     self.assertEqual(set(utils.getHosts()), set(hosts))
Exemple #10
0
 def test_getWorkerQteFile(self):
     self.assertEqual(utils.getWorkerQte(utils.getHosts("hostfilesim.txt")),
                      16)
Exemple #11
0
 def test_getWorkerQteSGE(self):
     os.environ.update(SGE_ENV)
     self.assertEqual(utils.getWorkerQte(utils.getHosts()), 16)
Exemple #12
0
 def test_getHostsFile(self):
     self.assertEqual(set(utils.getHosts("hostfilesim.txt")), set(hosts))