def test_get_ipython_dir_3():
"""test_get_ipython_dir_3, Testcase to see if we can call get_ipython_dir without Exceptions."""
genutils.get_home_dir = lambda: "someplace"
os.name = "nt"
ipdir = genutils.get_ipython_dir()
nt.assert_equal(ipdir,
os.path.abspath(os.path.join("someplace", "_ipython")))
def resolve_file_path(self, filename, ipythondir=None):
"""Resolve filenames into absolute paths.
This function looks in the following directories in order:
1. In the current working directory or by absolute path with ~ expanded
2. In ipythondir if that is set
3. In the IPYTHONDIR environment variable if it exists
4. In the ~/.ipython directory
Note: The IPYTHONDIR is also used by the trunk version of IPython so
changing it will also affect it was well.
"""
# In cwd or by absolute path with ~ expanded
trythis = os.path.expanduser(filename)
if os.path.isfile(trythis):
return trythis
# In ipythondir if it is set
if ipythondir is not None:
trythis = pjoin(ipythondir, filename)
if os.path.isfile(trythis):
return trythis
trythis = pjoin(get_ipython_dir(), filename)
if os.path.isfile(trythis):
return trythis
return None
def resolve_file_path(self, filename, ipythondir = None):
"""Resolve filenames into absolute paths.
This function looks in the following directories in order:
1. In the current working directory or by absolute path with ~ expanded
2. In ipythondir if that is set
3. In the IPYTHONDIR environment variable if it exists
4. In the ~/.ipython directory
Note: The IPYTHONDIR is also used by the trunk version of IPython so
changing it will also affect it was well.
"""
# In cwd or by absolute path with ~ expanded
trythis = os.path.expanduser(filename)
if os.path.isfile(trythis):
return trythis
# In ipythondir if it is set
if ipythondir is not None:
trythis = pjoin(ipythondir, filename)
if os.path.isfile(trythis):
return trythis
trythis = pjoin(get_ipython_dir(), filename)
if os.path.isfile(trythis):
return trythis
return None
def test_filefind():
"""Various tests for filefind"""
f = tempfile.NamedTemporaryFile()
print 'fname:', f.name
alt_dirs = genutils.get_ipython_dir()
t = genutils.filefind(f.name, alt_dirs)
print 'found:', t
def test_filefind():
"""Various tests for filefind"""
f = tempfile.NamedTemporaryFile()
print 'fname:',f.name
alt_dirs = genutils.get_ipython_dir()
t = genutils.filefind(f.name,alt_dirs)
print 'found:',t
def _getEngineInfoLogFile(self):
# Store all logs inside the ipython directory
ipdir = get_ipython_dir()
pjoin = os.path.join
logdir_base = pjoin(ipdir,'log')
if not os.path.isdir(logdir_base):
os.makedirs(logdir_base)
logfile = os.path.join(logdir_base,'ipcontroller-%s-engine-info.log' % os.getpid())
return logfile
def _getEngineInfoLogFile(self):
# Store all logs inside the ipython directory
ipdir = get_ipython_dir()
pjoin = os.path.join
logdir_base = pjoin(ipdir, 'log')
if not os.path.isdir(logdir_base):
os.makedirs(logdir_base)
logfile = os.path.join(logdir_base,
'ipcontroller-%s-engine-info.log' % os.getpid())
return logfile
def configuration(parent_package='',top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('utils', parent_package, top_path)
config.add_data_dir('tests')
try:
# If the user has IPython installed, this will install the
# nipype profile under their '~/.ipython' directory so they
# can launch ipython with 'ipython -p nipype' and the traits
# completer will be enabled by default.
from IPython.genutils import get_ipython_dir
pth = get_ipython_dir()
#config.data_files = [(pth, [join('nipype','utils','ipy_profile_nipype.py')])]
except ImportError:
# Don't do anything if they haven't installed IPython
pass
return config
from IPython.kernel.fcutil import Tub, UnauthenticatedTub
from IPython.kernel.core.config import config_manager as core_config_manager
from IPython.config.cutils import import_item
from IPython.kernel.engineservice import EngineService
# Create various ipython directories if they don't exist.
# This must be done before IPython.kernel.config is imported.
from IPython.iplib import user_setup
from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
if os.name == 'posix':
rc_suffix = ''
else:
rc_suffix = '.ini'
user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
get_log_dir()
get_security_dir()
from IPython.kernel.config import config_manager as kernel_config_manager
from IPython.kernel.engineconnector import EngineConnector
#-------------------------------------------------------------------------------
# Code
#-------------------------------------------------------------------------------
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
def test_get_ipython_dir_3():
"""test_get_ipython_dir_3, Testcase to see if we can call get_ipython_dir without Exceptions."""
genutils.get_home_dir = lambda : "someplace"
os.name = "nt"
ipdir = genutils.get_ipython_dir()
nt.assert_equal(ipdir, os.path.abspath(os.path.join("someplace", "_ipython")))
def test_get_ipython_dir_1():
"""test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
env['IPYTHONDIR'] = "someplace/.ipython"
ipdir = genutils.get_ipython_dir()
nt.assert_equal(ipdir, os.path.abspath("someplace/.ipython"))
def write_default_config_file(self):
ipdir = get_ipython_dir()
fname = pjoin(ipdir, self.filename)
if not os.path.isfile(fname):
print "Writing the configuration file to: " + fname
self.write_config_obj_to_file(fname)
def clusterRemote(opt,arg):
"""Start a remote cluster over SSH"""
# B. Granger, 9/3/08
# The launching of a remote cluster using SSH and a clusterfile
# is broken. Because it won't be fixed before the 0.9 release,
# we are removing it. For now, we just print a message to the
# user and abort.
print """The launching of a remote IPython cluster using SSL
and a clusterfile has been removed in this release.
It has been broken for a while and we are in the process
of building a new process management system that will be
used to provide a more robust way of starting an IPython
cluster.
For now remote clusters have to be launched using ipcontroller
and ipengine separately.
"""
sys.exit(1)
# Load the remote cluster configuration
clConfig = {}
execfile(opt.clusterfile,clConfig)
contConfig = clConfig['controller']
engConfig = clConfig['engines']
# Determine where to find sshx:
sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
# Store all logs inside the ipython directory
ipdir = get_ipython_dir()
pjoin = os.path.join
logfile = opt.logfile
if logfile is None:
logdir_base = pjoin(ipdir,'log')
ensureDir(logdir_base)
logfile = pjoin(logdir_base,'ipcluster')
# Append this script's PID to the logfile name always
logfile = '%s-%s' % (logfile,os.getpid())
print 'Starting controller:'
# Controller data:
xsys = os.system
contHost = contConfig['host']
contLog = '%s-con-%s-' % (logfile,contHost)
cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
(contHost,sshx,contLog)
#print 'cmd:<%s>' % cmd # dbg
xsys(cmd)
time.sleep(2)
print 'Starting engines: '
for engineHost,engineData in engConfig.iteritems():
if isinstance(engineData,int):
numEngines = engineData
else:
raise NotImplementedError('port configuration not finished for engines')
print 'Sarting %d engines on %s' % (numEngines,engineHost)
engLog = '%s-eng-%s-' % (logfile,engineHost)
for i in range(numEngines):
cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
(engineHost,sshx,contHost,engLog)
#print 'cmd:<%s>' % cmd # dbg
xsys(cmd)
# Wait after each host a little bit
time.sleep(1)
startMsg(contConfig['host'])
def clusterLocal(opt,arg):
"""Start a cluster on the local machine."""
# Store all logs inside the ipython directory
ipdir = get_ipython_dir()
pjoin = os.path.join
logfile = opt.logfile
if logfile is None:
logdir_base = pjoin(ipdir,'log')
ensureDir(logdir_base)
logfile = pjoin(logdir_base,'ipcluster-')
print 'Starting controller:',
controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
print 'Controller PID:',controller.pid
print 'Starting engines: ',
time.sleep(5)
englogfile = '%s%s-' % (logfile,controller.pid)
mpi = opt.mpi
if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi',
mpi, '--logfile',englogfile])]
# engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
else: # do what we would normally do
engines = [ Popen(['ipengine','--logfile',englogfile])
for i in range(opt.n) ]
eids = [e.pid for e in engines]
print 'Engines PIDs: ',eids
print 'Log files: %s*' % englogfile
proc_ids = eids + [controller.pid]
procs = engines + [controller]
grpid = os.getpgrp()
try:
startMsg('127.0.0.1')
print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
print
print 'kill -INT',grpid
print
try:
while True:
time.sleep(5)
except:
pass
finally:
print 'Stopping cluster. Cleaning up...'
cleanup(stop,controller,engines)
for i in range(4):
time.sleep(i+2)
nZombies = numAlive(controller,engines)
if nZombies== 0:
print 'OK: All processes cleaned up.'
break
print 'Trying again, %d processes did not stop...' % nZombies
cleanup(kill,controller,engines)
if numAlive(controller,engines) == 0:
print 'OK: All processes cleaned up.'
break
else:
print '*'*75
print 'ERROR: could not kill some processes, try to do it',
print 'manually.'
zombies = []
if controller.returncode is None:
print 'Controller is alive: pid =',controller.pid
zombies.append(controller.pid)
liveEngines = [ e for e in engines if e.returncode is None ]
for e in liveEngines:
print 'Engine is alive: pid =',e.pid
zombies.append(e.pid)
print
print 'Zombie summary:',' '.join(map(str,zombies))
def test_get_ipython_dir():
"""Make sure we can get the ipython directory."""
ipdir = genutils.get_ipython_dir()
from IPython.genutils import (
get_ipython_dir,
get_log_dir,
get_security_dir,
num_cpus
)
from IPython.kernel.fcutil import have_crypto
# Create various ipython directories if they don't exist.
# This must be done before IPython.kernel.config is imported.
from IPython.iplib import user_setup
if os.name == 'posix':
rc_suffix = ''
else:
rc_suffix = '.ini'
user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
get_log_dir()
get_security_dir()
from IPython.kernel.config import config_manager as kernel_config_manager
from IPython.kernel.error import SecurityError, FileTimeoutError
from IPython.kernel.fcutil import have_crypto
from IPython.kernel.twistedutil import gatherBoth, wait_for_file
from IPython.kernel.util import printer
#-----------------------------------------------------------------------------
# General process handling code
#-----------------------------------------------------------------------------
class ProcessStateError(Exception):
def get_args():
base_parser = argparse.ArgumentParser(add_help=False)
base_parser.add_argument(
'-r',
action='store_true',
dest='r',
help=
'try to reuse FURL files. Use with --client-port and --engine-port')
base_parser.add_argument(
'--client-port',
type=int,
dest='client_port',
help='the port the controller will listen on for client connections',
default=0)
base_parser.add_argument(
'--engine-port',
type=int,
dest='engine_port',
help='the port the controller will listen on for engine connections',
default=0)
base_parser.add_argument('-x',
action='store_true',
dest='x',
help='turn off client security')
base_parser.add_argument('-y',
action='store_true',
dest='y',
help='turn off engine security')
base_parser.add_argument(
"--logdir",
type=str,
dest="logdir",
help="directory to put log files (default=$IPYTHONDIR/log)",
default=pjoin(get_ipython_dir(), 'log'))
base_parser.add_argument("-n",
"--num",
type=int,
dest="n",
default=2,
help="the number of engines to start")
parser = argparse.ArgumentParser(
description='IPython cluster startup. This starts a controller and\
engines using various approaches. Use the IPYTHONDIR environment\
variable to change your IPython directory from the default of\
.ipython or _ipython. The log and security subdirectories of your\
IPython directory will be used by this script for log files and\
security files.')
subparsers = parser.add_subparsers(
help='available cluster types. For help, do "ipcluster TYPE --help"')
parser_local = subparsers.add_parser('local',
help='run a local cluster',
parents=[base_parser])
parser_local.set_defaults(func=main_local)
parser_mpirun = subparsers.add_parser(
'mpirun',
help='run a cluster using mpirun (mpiexec also works)',
parents=[base_parser])
parser_mpirun.add_argument(
"--mpi",
type=str,
dest="mpi", # Don't put a default here to allow no MPI support
help="how to call MPI_Init (default=mpi4py)")
parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun')
parser_mpiexec = subparsers.add_parser(
'mpiexec',
help='run a cluster using mpiexec (mpirun also works)',
parents=[base_parser])
parser_mpiexec.add_argument(
"--mpi",
type=str,
dest="mpi", # Don't put a default here to allow no MPI support
help="how to call MPI_Init (default=mpi4py)")
parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec')
parser_pbs = subparsers.add_parser('pbs',
help='run a pbs cluster',
parents=[base_parser])
parser_pbs.add_argument('--pbs-script',
type=str,
dest='pbsscript',
help='PBS script template',
default='pbs.template')
parser_pbs.set_defaults(func=main_pbs)
parser_ssh = subparsers.add_parser(
'ssh',
help='run a cluster using ssh, should have ssh-keys setup',
parents=[base_parser])
parser_ssh.add_argument(
'--clusterfile',
type=str,
dest='clusterfile',
help='python file describing the cluster',
default='clusterfile.py',
)
parser_ssh.add_argument('--sshx',
type=str,
dest='sshx',
help='sshx launcher helper')
parser_ssh.set_defaults(func=main_ssh)
args = parser.parse_args()
return args
def test_get_ipython_dir():
"""Make sure we can get the ipython directory."""
ipdir = genutils.get_ipython_dir()
def write_default_config_file(self):
ipdir = get_ipython_dir()
fname = pjoin(ipdir, self.filename)
if not os.path.isfile(fname):
print "Writing the configuration file to: " + fname
self.write_config_obj_to_file(fname)
def get_args():
base_parser = argparse.ArgumentParser(add_help=False)
base_parser.add_argument(
'-r',
action='store_true',
dest='r',
help='try to reuse FURL files. Use with --client-port and --engine-port'
)
base_parser.add_argument(
'--client-port',
type=int,
dest='client_port',
help='the port the controller will listen on for client connections',
default=0
)
base_parser.add_argument(
'--engine-port',
type=int,
dest='engine_port',
help='the port the controller will listen on for engine connections',
default=0
)
base_parser.add_argument(
'-x',
action='store_true',
dest='x',
help='turn off client security'
)
base_parser.add_argument(
'-y',
action='store_true',
dest='y',
help='turn off engine security'
)
base_parser.add_argument(
"--logdir",
type=str,
dest="logdir",
help="directory to put log files (default=$IPYTHONDIR/log)",
default=pjoin(get_ipython_dir(),'log')
)
base_parser.add_argument(
"-n",
"--num",
type=int,
dest="n",
default=2,
help="the number of engines to start"
)
parser = argparse.ArgumentParser(
description='IPython cluster startup. This starts a controller and\
engines using various approaches. Use the IPYTHONDIR environment\
variable to change your IPython directory from the default of\
.ipython or _ipython. The log and security subdirectories of your\
IPython directory will be used by this script for log files and\
security files.'
)
subparsers = parser.add_subparsers(
help='available cluster types. For help, do "ipcluster TYPE --help"')
parser_local = subparsers.add_parser(
'local',
help='run a local cluster',
parents=[base_parser]
)
parser_local.set_defaults(func=main_local)
parser_mpirun = subparsers.add_parser(
'mpirun',
help='run a cluster using mpirun (mpiexec also works)',
parents=[base_parser]
)
parser_mpirun.add_argument(
"--mpi",
type=str,
dest="mpi", # Don't put a default here to allow no MPI support
help="how to call MPI_Init (default=mpi4py)"
)
parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun')
parser_mpiexec = subparsers.add_parser(
'mpiexec',
help='run a cluster using mpiexec (mpirun also works)',
parents=[base_parser]
)
parser_mpiexec.add_argument(
"--mpi",
type=str,
dest="mpi", # Don't put a default here to allow no MPI support
help="how to call MPI_Init (default=mpi4py)"
)
parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec')
parser_pbs = subparsers.add_parser(
'pbs',
help='run a pbs cluster',
parents=[base_parser]
)
parser_pbs.add_argument(
'--pbs-script',
type=str,
dest='pbsscript',
help='PBS script template',
default='pbs.template'
)
parser_pbs.set_defaults(func=main_pbs)
parser_ssh = subparsers.add_parser(
'ssh',
help='run a cluster using ssh, should have ssh-keys setup',
parents=[base_parser]
)
parser_ssh.add_argument(
'--clusterfile',
type=str,
dest='clusterfile',
help='python file describing the cluster',
default='clusterfile.py',
)
parser_ssh.add_argument(
'--sshx',
type=str,
dest='sshx',
help='sshx launcher helper'
)
parser_ssh.set_defaults(func=main_ssh)
args = parser.parse_args()
return args
def test_get_ipython_dir_1():
"""test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
env['IPYTHONDIR'] = "someplace/.ipython"
ipdir = genutils.get_ipython_dir()
nt.assert_equal(ipdir, os.path.abspath("someplace/.ipython"))