def activate(profile, mixed=False):
dname = os.path.dirname(__file__)
pdir = locate_profile(profile)
jc = JSONFileConfigLoader('ipython_notebook_config.json', pdir)
try:
config = jc.load_config()
except (ConfigFileNotFound, ValueError) as e:
config = Config()
if 'NotebookApp' in config:
if ('tornado_settings'
in config['NotebookApp']) or ('contents_manager_class'
in config['NotebookApp']):
# TODO, manually merge tornado settin if exist
# but cannot do anythin automatically if contents_manager_calss is set
raise ValueError(
'You already got some configuration that will conflict with google drive. Bailin out'
)
if mixed:
drive_config = JSONFileConfigLoader('mixed_contents.json',
dname).load_config()
else:
drive_config = JSONFileConfigLoader('ipython_notebook_config.json',
dname).load_config()
config.merge(drive_config)
print('Activating Google Drive integration for profile "%s"' % profile)
config['nbformat'] = 1
with io.open(os.path.join(pdir, 'ipython_notebook_config.json'),
'w',
encoding='utf-8') as f:
f.write(cast_unicode_py2(json.dumps(config, indent=2)))
def test_sys_path_profile_dir():
"""test that sys.path doesn't get messed up when `--profile-dir` is specified"""
with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
stdout, stderr = assemble_output(kc.iopub_channel)
nt.assert_equal(stdout, "''\n")
def edit_ipython_profile(spark_home, spark_python, py4j):
"""Adds a startup file to the current IPython profile to import pyspark.
The startup file sets the required environment variables and imports pyspark.
Parameters
----------
spark_home : str
Path to Spark installation.
spark_python : str
Path to python subdirectory of Spark installation.
py4j : str
Path to py4j library.
"""
from IPython import get_ipython
ip = get_ipython()
if ip:
profile_dir = ip.profile_dir.location
else:
from IPython.utils.path import locate_profile
profile_dir = locate_profile()
startup_file_loc = os.path.join(profile_dir, "startup", "findspark.py")
with open(startup_file_loc, 'w') as startup_file:
#Lines of code to be run when IPython starts
startup_file.write("import sys, os\n")
startup_file.write("os.environ['SPARK_HOME'] = '" + spark_home + "'\n")
startup_file.write("sys.path[:0] = " + str([spark_python, py4j]) +
"\n")
startup_file.write("import pyspark\n")
def customize_settings():
from IPython.display import display, HTML
from IPython.html.services.config import ConfigManager
from IPython.utils.path import locate_profile
cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))
cm.update('livereveal', {'width': 1024,'height': 768,})
display(HTML(open("notebook_style.css").read()))
def edit_ipython_profile(spark_home, spark_python, py4j):
"""Adds a startup file to the current IPython profile to import pyspark.
The startup file sets the required enviornment variables and imports pyspark.
Parameters
----------
spark_home : str
Path to Spark installation.
spark_python : str
Path to python subdirectory of Spark installation.
py4j : str
Path to py4j library.
"""
ip = get_ipython()
if ip:
profile_dir = ip.profile_dir.location
else:
from IPython.utils.path import locate_profile
profile_dir = locate_profile()
startup_file_loc = os.path.join(profile_dir, "startup", "findspark.py")
with open(startup_file_loc, 'w') as startup_file:
#Lines of code to be run when IPython starts
startup_file.write("import sys, os\n")
startup_file.write("os.environ['SPARK_HOME'] = '" + spark_home + "'\n")
startup_file.write("sys.path[:0] = " + str([spark_python, py4j]) + "\n")
startup_file.write("import pyspark\n")
def install_ipython_js():
"""Render GLmol library and install into ipython profile static include directory.
Resolves current ipython profile directory and writes the rendered library as:
'<ipython_profile_dir>/static/glmol/GLmol.full.devel.js'.
returns - Script url for use in the notebook."""
base_path = "static/glmol"
base_name = "GLmol.full.devel.js"
from IPython.utils.path import locate_profile
base_dir = path.join(locate_profile(), base_path)
if path.exists(base_dir):
logger.info("Removing existing glmol: %s", base_dir)
shutil.rmtree(base_dir)
os.makedirs(base_dir)
output_file = path.join(base_dir, base_name)
logger.info("Writing glmol: %s", output_file)
with open(output_file, "w") as o:
o.write(render_js())
return path.join(base_path, base_name)
def _init_index(self, reset=False):
index_path = os.path.join(locate_profile(), "index")
# clear out old index if requested
if reset:
shutil.rmtree(index_path, True)
# make sure there's a path to store the index data
if not os.path.exists(index_path):
os.makedirs(index_path)
if not exists_in(index_path):
# create an index with the current schema
schema = Schema(
basename=TEXT(stored=True, field_boost=5.0),
dirname=ID(stored=True),
path=ID(stored=True, unique=True),
content=TEXT(stored=False),
time=STORED,
)
self.ix = create_in(index_path, schema)
else:
# open the existing index
self.ix = open_dir(index_path)
# build a query parser based on the current schema
self.query_parser = MultifieldParser(["content", "basename", "dirname"], self.ix.schema)
def activate(profile,mixed=False):
dname = os.path.dirname(__file__)
pdir = locate_profile(profile)
jc = JSONFileConfigLoader('ipython_notebook_config.json',pdir)
try:
config = jc.load_config();
except (ConfigFileNotFound,ValueError) as e:
config = Config()
if 'NotebookApp' in config:
if ('tornado_settings' in config['NotebookApp']) or ('contents_manager_class' in config['NotebookApp']):
# TODO, manually merge tornado settin if exist
# but cannot do anythin automatically if contents_manager_calss is set
raise ValueError('You already got some configuration that will conflict with google drive. Bailin out')
if mixed :
drive_config = JSONFileConfigLoader('mixed_contents.json', dname).load_config()
else :
drive_config = JSONFileConfigLoader('ipython_notebook_config.json', dname).load_config()
config.merge(drive_config)
print('Activating Google Drive integration for profile "%s"' % profile)
config['nbformat'] = 1
with io.open(os.path.join(pdir,'ipython_notebook_config.json'),'w', encoding='utf-8') as f:
f.write(cast_unicode_py2(json.dumps(config, indent=2)))
def test_sys_path_profile_dir():
"""test that sys.path doesn't get messed up when `--profile-dir` is specified"""
with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
stdout, stderr = assemble_output(kc.iopub_channel)
nt.assert_equal(stdout, "''\n")
def _create_base_ipython_dirs():
"""Create default user directories to prevent potential race conditions downstream.
"""
utils.safe_makedir(get_ipython_dir())
ProfileDir.create_profile_dir_by_name(get_ipython_dir())
utils.safe_makedir(os.path.join(get_ipython_dir(), "db"))
utils.safe_makedir(os.path.join(locate_profile(), "db"))
def install_mathjax(tag='v2.0', replace=False, dest=None):
"""Download and install MathJax for offline use.
You can use this to install mathjax to a location on your static file
path. This includes the `static` directory within your IPython profile,
which is the default location for this install.
MathJax is a ~15MB download, and ~150MB installed.
Parameters
----------
replace : bool [False]
Whether to remove and replace an existing install.
tag : str ['v2.0']
Which tag to download. Default is 'v2.0', the current stable release,
but alternatives include 'v1.1' and 'master'.
dest : path
The path to the directory in which mathjax will be installed.
The default is `IPYTHONDIR/profile_default/static`.
dest must be on your notebook static_path when you run the notebook server.
The default location works for this.
"""
mathjax_url = "https://github.com/mathjax/MathJax/tarball/%s" % tag
if dest is None:
dest = os.path.join(locate_profile('default'), 'static')
if not os.path.exists(dest):
os.mkdir(dest)
static = dest
dest = os.path.join(static, 'mathjax')
# check for existence and permissions
if not os.access(static, os.W_OK):
raise IOError("Need have write access to %s" % static)
if os.path.exists(dest):
if replace:
if not os.access(dest, os.W_OK):
raise IOError("Need have write access to %s" % dest)
print "removing previous MathJax install"
shutil.rmtree(dest)
else:
print "offline MathJax apparently already installed"
return
# download mathjax
print "Downloading mathjax source from %s ..." % mathjax_url
response = urllib2.urlopen(mathjax_url)
print "done"
# use 'r|gz' stream mode, because socket file-like objects can't seek:
tar = tarfile.open(fileobj=response.fp, mode='r|gz')
topdir = tar.firstmember.path
print "Extracting to %s" % dest
tar.extractall(static)
# it will be mathjax-MathJax-<sha>, rename to just mathjax
os.rename(os.path.join(static, topdir), dest)
def install(use_symlink=False, profile="default"):
# Install the livereveal code.
install_nbextension(livereveal_dir, symlink=use_symlink, overwrite=use_symlink)
# Enable the extension in the given profile.
profile_dir = locate_profile(profile)
custom_js = os.path.join(profile_dir, "static", "custom", "custom.js")
add_if_not_in_custom_js(custom_js, "LIVE_REVEAL", custom_js_entry)
def main():
# This main is just simple enough that it is not worth the
# complexity of argparse
# What directory is mathjax in?
parser = argparse.ArgumentParser(description="""Install mathjax from internet or local archive""")
parser.add_argument("-p", "--profile", default="default", help="profile to install MathJax to (default is default)")
parser.add_argument("-i", "--install-dir", help="custom installation directory")
parser.add_argument(
"-d", "--dest", action="store_true", help="print where current mathjax would be installed and exit"
)
parser.add_argument(
"-r", "--replace", action="store_true", help="Whether to replace current mathjax if it already exists"
)
parser.add_argument("-t", "--test", action="store_true")
parser.add_argument("tarball", help="the local tar/zip-ball containing mathjax", nargs="?", metavar="tarball")
pargs = parser.parse_args()
if pargs.install_dir:
# Explicit install_dir overrides profile
dest = pargs.install_dir
else:
profile = pargs.profile
dest = os.path.join(locate_profile(profile), "static", "mathjax")
if pargs.dest:
print(dest)
return
# remove/replace existing mathjax?
if pargs.replace:
replace = True
else:
replace = False
# undocumented test interface
if pargs.test:
return test_func(replace, dest)
# do it
if pargs.tarball:
fname = pargs.tarball
# automatically detect zip/tar - could do something based
# on file content, but really not cost-effective here.
if fname.endswith(".zip"):
extractor = extract_zip
else:
extractor = extract_tar
# do it
install_mathjax(file=open(fname, "r"), replace=replace, extractor=extractor, dest=dest)
else:
install_mathjax(replace=replace, dest=dest)
def client_directory():
"""
Returns the directory in which to download client files.
This will vary between Jupyter 3 and 4.
"""
if ipy_major_version == 3:
return locate_profile() + '/static/genepattern'
else:
return jupyter_core.paths.jupyter_config_dir() + '/custom/genepattern'
def get_url_file(profile, cluster_id):
url_file = "ipcontroller-{0}-client.json".format(cluster_id)
if os.path.isdir(profile) and os.path.isabs(profile):
# Return full_path if one is given
return os.path.join(profile, "security", url_file)
return os.path.join(locate_profile(profile), "security", url_file)
def get_url_file(profile, cluster_id):
url_file = "ipcontroller-{0}-client.json".format(cluster_id)
if os.path.isdir(profile) and os.path.isabs(profile):
# Return full_path if one is given
return os.path.join(profile, "security", url_file)
return os.path.join(locate_profile(profile), "security", url_file)
def install(use_symlink=False, profile='default', enable=True):
# Install the livereveal code.
install_nbextension(livereveal_dir, symlink=use_symlink,
overwrite=use_symlink, user=True)
if enable:
# Enable the extension in the given profile.
profile_dir = locate_profile(profile)
cm = ConfigManager(profile_dir=profile_dir)
cm.update('notebook', {"load_extensions": {"livereveal/main": True}})
def customize_settings():
from IPython.display import display, HTML
from IPython.html.services.config import ConfigManager
from IPython.utils.path import locate_profile
cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))
cm.update('livereveal', {
'width': 1024,
'height': 768,
})
display(HTML(open("notebook_style.css").read()))
def _find_profile(profile_name):
from IPython.utils.path import locate_profile, get_ipython_dir
try:
profile_path = locate_profile(profile_name)
exists = True
except IOError:
profile_path = os.path.join(get_ipython_dir(),
"profile_%s" % profile_name)
exists = False
return profile_path, exists
def install(extension, use_symlink=False, profile='default', enable=True):
# Install the livereveal code.
ext_dir = os.path.join(base_dir, extension)
install_nbextension(ext_dir, symlink=use_symlink,
overwrite=use_symlink, user=True)
# Enable/Disable the extension in the given profile.
profile_dir = locate_profile(profile)
cm = ConfigManager(profile_dir=profile_dir)
cm.update('notebook', {"load_extensions": {extension + "/main": enable}})
def install(use_symlink=False, profile='default'):
# Install the livereveal code.
install_nbextension(livereveal_dir,
symlink=use_symlink,
overwrite=use_symlink)
# Enable the extension in the given profile.
profile_dir = locate_profile(profile)
custom_js = os.path.join(profile_dir, 'static', 'custom', 'custom.js')
add_if_not_in_custom_js(custom_js, 'LIVE_REVEAL', custom_js_entry)
def _get_hist_file_name(self, profile="default"):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return os.path.join(locate_profile(profile), "history.sqlite")
def install(use_symlink=False, profile='default', enable=True):
# Install the livereveal code.
install_nbextension(livereveal_dir,
symlink=use_symlink,
overwrite=use_symlink,
user=True)
if enable:
# Enable the extension in the given profile.
profile_dir = locate_profile(profile)
cm = ConfigManager(profile_dir=profile_dir)
cm.update('notebook', {"load_extensions": {"livereveal/main": True}})
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._start_jython()
try:
self.hist_file = os.path.join(locate_profile(),'jython_kernel.hist')
except:
self.hist_file = None
self.log.warn('No default profile found, history unavailable')
self.max_hist_cache = 1000
self.hist_cache = []
def _get_hist_file_name(self, profile='default'):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return os.path.join(locate_profile(profile), 'history.sqlite')
def install(extension, use_symlink=False, profile='default', enable=True):
# Install the livereveal code.
ext_dir = os.path.join(base_dir, extension)
install_nbextension(ext_dir,
symlink=use_symlink,
overwrite=use_symlink,
user=True)
# Enable/Disable the extension in the given profile.
profile_dir = locate_profile(profile)
cm = ConfigManager(profile_dir=profile_dir)
cm.update('notebook', {"load_extensions": {extension + "/main": enable}})
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._start_jython()
try:
self.hist_file = os.path.join(locate_profile(),'jython_kernel.hist')
except:
self.hist_file = None
self.log.warn('No default profile found, history unavailable')
self.max_hist_cache = 1000
self.hist_cache = []
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._start_ncl()
try:
self.hist_file = os.path.join(locate_profile(),'ncl_kernel.hist')
except:
self.hist_file = None
self.log.warn('No default profile found, history unavailable')
self.max_hist_cache = 1000
self.hist_cache = []
#self._default_matches=cPickle.load(open('data/inbuilt_list', 'rb'))
self._default_matches=self.inbuiltlist()
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._start_ncl()
self.pexpect_version=pexpect.__version__
try:
self.hist_file = os.path.join(locate_profile(),'ncl_kernel.hist')
except:
self.hist_file = None
self.log.warn('No default profile found, history unavailable')
self.max_hist_cache = 1000
self.hist_cache = []
#self._default_matches=cPickle.load(open('data/inbuilt_list', 'rb'))
self._default_matches=self.inbuiltlist()
def create_throwaway_profile(
executable, ssh_client=None, in_work_dir=False, work_dir=None
):
profile = str(uuid.uuid1())
if in_work_dir:
if work_dir is None:
raise ValueError
profile = os.path.join(
work_dir, "profile_{}".format(profile)
)
profile_dir = profile
cmd = [
executable, "-E", "-c",
"from IPython import start_ipython; start_ipython()",
"profile", "create",
profile if not in_work_dir or work_dir is None
else "--profile-dir={}".format(profile),
"--parallel"
]
if ssh_client is None:
subprocess.check_call(cmd)
if not in_work_dir:
profile_dir = locate_profile(profile)
else:
remote_cmd = ' '.join([
'"{}"'.format(item) if ' ' in item else item
for item in cmd
])
ssh_client.exec_command(
'source ~/.profile; ' + remote_cmd
)
if not in_work_dir:
remote_cmd = (
'{0} -E -c '
'"from IPython.utils.path import locate_profile; '
'import os; '
'print(\'{1}\' if os.path.isdir(\'{1}\') and os.path.isabs(\'{1}\') '
'else locate_profile(\'{1}\'))"'
).format(executable, profile)
profile_dir = ssh_client.exec_command(
'source ~/.profile > /dev/null; ' + remote_cmd
)[1].read().decode('utf-8').strip()
return profile, profile_dir
def load_ipython_extension(ipython):
# use the configured working directory if we can find it
work_dir = None
for filename in scandir.scandir(os.path.join(locate_profile(), 'security')):
if filename.name.startswith('nbserver-') and filename.name.endswith('.json'):
with open(filename.path, 'r') as fh:
nbserver = json.load(fh)
work_dir = nbserver['notebook_dir']
break
if work_dir is None:
# fall back on an environment variable or ultimately the pwd
work_dir = os.getenv('WORK', '.')
loader.enable(work_dir)
ipython.push({'load_notebook': loader.load_notebook})
ipython.register_magics(InjectMagic(ipython))
def delete_profile(profile):
MAX_TRIES = 10
dir_to_remove = locate_profile(profile)
if os.path.exists(dir_to_remove):
num_tries = 0
while True:
try:
shutil.rmtree(dir_to_remove)
break
except OSError:
if num_tries > MAX_TRIES:
raise
time.sleep(5)
num_tries += 1
else:
raise ValueError("Cannot find {0} to remove, " "something is wrong.".format(dir_to_remove))
def delete_profile(profile):
MAX_TRIES = 10
dir_to_remove = locate_profile(profile)
if os.path.exists(dir_to_remove):
num_tries = 0
while True:
try:
shutil.rmtree(dir_to_remove)
break
except OSError:
if num_tries > MAX_TRIES:
raise
time.sleep(5)
num_tries += 1
else:
raise ValueError("Cannot find {0} to remove, "
"something is wrong.".format(dir_to_remove))
def get_metadata_engine(other_engine):
"""Create and return an SA engine for which will be used for
storing ipydb db metadata about the input engine.
Args:
other_engine - SA engine for which we will be storing metadata for.
Returns:
tuple (dbname, sa_engine). dbname is a unique key for the input
other_engine. sa_engine is the SA engine that will be used for storing
metadata about `other_engine`
"""
path = os.path.join(locate_profile(), 'ipydb')
if not os.path.exists(path):
os.makedirs(path)
dbfilename = get_db_filename(other_engine)
dburl = u'sqlite:////%s' % os.path.join(path, dbfilename)
return dbfilename, sa.create_engine(dburl)
import os
import shutil
import sys
import tarfile
import urllib2
import zipfile
from IPython.utils.path import locate_profile
from IPython.external import argparse
#-----------------------------------------------------------------------------
#
#-----------------------------------------------------------------------------
# Where mathjax will be installed.
static = os.path.join(locate_profile('default'), 'static')
default_dest = os.path.join(static, 'mathjax')
##
# Test for access to install mathjax.
def check_perms(dest, replace=False):
parent = os.path.abspath(os.path.join(dest, os.path.pardir))
components = dest.split(os.path.sep)
subpaths = [
os.path.sep + os.path.sep.join(components[1:i])
for i in range(1, len(components))
]
from IPython.html.services.config import ConfigManager
from IPython.utils.path import locate_profile
cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))
cm.update('livereveal', {
'theme': 'beige',
'transition': 'fade',
'start_slideshow_at': 'selected',
})
def main() :
# This main is just simple enough that it is not worth the
# complexity of argparse
# What directory is mathjax in?
parser = argparse.ArgumentParser(
description="""Install mathjax from internet or local archive""",
)
parser.add_argument(
'-p',
'--profile',
default='default',
help='profile to install MathJax to (default is default)')
parser.add_argument(
'-i',
'--install-dir',
help='custom installation directory')
parser.add_argument(
'-d',
'--dest',
action='store_true',
help='print where current mathjax would be installed and exit')
parser.add_argument(
'-r',
'--replace',
action='store_true',
help='Whether to replace current mathjax if it already exists')
parser.add_argument(
'-t',
'--test',
action='store_true')
parser.add_argument('tarball',
help="the local tar/zip-ball containing mathjax",
nargs='?',
metavar='tarball')
pargs = parser.parse_args()
if pargs.install_dir:
# Explicit install_dir overrides profile
dest = pargs.install_dir
else:
profile = pargs.profile
dest = os.path.join(locate_profile(profile), 'static', 'mathjax')
if pargs.dest :
print dest
return
# remove/replace existing mathjax?
if pargs.replace :
replace = True
else :
replace = False
# undocumented test interface
if pargs.test :
return test_func( replace, dest)
# do it
if pargs.tarball :
fname = pargs.tarball
# automatically detect zip/tar - could do something based
# on file content, but really not cost-effective here.
if fname.endswith('.zip') :
extractor = extract_zip
else :
extractor = extract_tar
# do it
install_mathjax(file=open(fname, "r"), replace=replace, extractor=extractor, dest=dest )
else:
install_mathjax(replace=replace, dest=dest)
import shutil
import sys
import tarfile
import urllib2
import zipfile
from IPython.utils.path import locate_profile
from IPython.external import argparse
#-----------------------------------------------------------------------------
#
#-----------------------------------------------------------------------------
# Where mathjax will be installed.
static = os.path.join(locate_profile('default'), 'static')
default_dest = os.path.join(static, 'mathjax')
##
# Test for access to install mathjax.
def check_perms(dest, replace=False):
parent = os.path.abspath(os.path.join(dest, os.path.pardir))
components = dest.split(os.path.sep)
subpaths = [ os.path.sep+os.path.sep.join(components[1:i]) for i in range(1,len(components))]
existing_path = filter(os.path.exists, subpaths)
last_writable = existing_path[-1]
if not os.access(last_writable, os.W_OK):
raise IOError("Need have write access to %s" % parent)
def main():
# This main is just simple enough that it is not worth the
# complexity of argparse
# What directory is mathjax in?
parser = argparse.ArgumentParser(
description="""Install mathjax from internet or local archive""", )
parser.add_argument(
'-p',
'--profile',
default='default',
help='profile to install MathJax to (default is default)')
parser.add_argument('-i',
'--install-dir',
help='custom installation directory')
parser.add_argument(
'-d',
'--dest',
action='store_true',
help='print where current mathjax would be installed and exit')
parser.add_argument(
'-r',
'--replace',
action='store_true',
help='Whether to replace current mathjax if it already exists')
parser.add_argument('-t', '--test', action='store_true')
parser.add_argument('tarball',
help="the local tar/zip-ball containing mathjax",
nargs='?',
metavar='tarball')
pargs = parser.parse_args()
if pargs.install_dir:
# Explicit install_dir overrides profile
dest = pargs.install_dir
else:
profile = pargs.profile
dest = os.path.join(locate_profile(profile), 'static', 'mathjax')
if pargs.dest:
print dest
return
# remove/replace existing mathjax?
if pargs.replace:
replace = True
else:
replace = False
# undocumented test interface
if pargs.test:
return test_func(replace, dest)
# do it
if pargs.tarball:
fname = pargs.tarball
# automatically detect zip/tar - could do something based
# on file content, but really not cost-effective here.
if fname.endswith('.zip'):
extractor = extract_zip
else:
extractor = extract_tar
# do it
install_mathjax(file=open(fname, "r"),
replace=replace,
extractor=extractor,
dest=dest)
else:
install_mathjax(replace=replace, dest=dest)
def ensure_controller(self, connect_only=False):
"""Make sure a controller is available, else start a local one.
"""
if self._client:
return self._client
if self.profile is None:
self._select_profile()
if self.profile is None:
return None
print "parallelflow: using IPython profile %r" % self.profile
try:
self._client = Client(profile=self.profile)
print "parallelflow: connected to controller"
return self._client
except error.TimeoutError:
print "parallelflow: timeout when connecting to controller"
if connect_only:
start_ctrl = False
elif qt_available:
res = QtGui.QMessageBox.question(
None,
"Start controller",
"Unable to connect to the configured IPython "
"controller. Do you want to start one?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
start_ctrl = res == QtGui.QMessageBox.Yes
else:
start_ctrl = True
except IOError:
print "parallelflow: didn't find a controller to connect to"
if connect_only:
start_ctrl = False
elif qt_available:
res = QtGui.QMessageBox.question(
None,
"Start controller",
"No controller is configured in this IPython profile. "
"Do you want to start one?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
start_ctrl = res == QtGui.QMessageBox.Yes
else:
start_ctrl = True
if start_ctrl:
ctrl_pid = os.path.join(
locate_profile(self.profile),
'pid',
'ipcontroller.pid')
if os.path.exists(ctrl_pid):
os.remove(ctrl_pid)
print "parallelflow: starting controller"
proc, code = self.start_process(
lambda: os.path.exists(ctrl_pid),
sys.executable,
'-m',
'IPython.parallel.apps.ipcontrollerapp',
'--profile=%s' % self.profile)
if code is not None:
if qt_available:
QtGui.QMessageBox.critical(
None,
"Error",
"Controller exited with code %d" % code)
print ("parallelflow: controller process exited with "
"code %d" % code)
return None
else:
self.started_controller = proc
print "parallelflow: controller started, connecting"
self._client = Client(profile=self.profile)
return self._client
return None
import sys
import tarfile
import urllib.request, urllib.error, urllib.parse
import zipfile
from IPython.utils.path import locate_profile
from IPython.external import argparse
# -----------------------------------------------------------------------------
#
# -----------------------------------------------------------------------------
# Where mathjax will be installed.
static = os.path.join(locate_profile("default"), "static")
default_dest = os.path.join(static, "mathjax")
##
# Test for access to install mathjax.
def check_perms(dest, replace=False):
parent = os.path.abspath(os.path.join(dest, os.path.pardir))
components = dest.split(os.path.sep)
subpaths = [os.path.sep + os.path.sep.join(components[1:i]) for i in range(1, len(components))]
existing_path = list(filter(os.path.exists, subpaths))
last_writable = existing_path[-1]
if not os.access(last_writable, os.W_OK):
def cluster_view(
queue, num_jobs,
sshhostname=None, sshuser=None, sshport=22, sshkey=None, sshpassword=None,
executable=sys.executable,
profile_in_work_dir=False,
cluster='Bcbio', work_dir=None,
scheduler='sge', cores_per_job=1, profile=None,
start_wait=16, extra_params=None, retries=None, direct=False
):
"""Provide a view on an ipython cluster for processing.
- scheduler: The type of cluster to start (lsf, sge, pbs, torque).
- num_jobs: Number of jobs to start.
- cores_per_job: The number of cores to use for each job.
- start_wait: How long to wait for the cluster to startup, in minutes.
Defaults to 16 minutes. Set to longer for slow starting clusters.
- retries: Number of retries to allow for failed tasks.
- ssh_client: A connected paramiko.client.SSHClient instance
- executable: the path to the executable
"""
if sshhostname is not None:
sshserver = sshhostname
if sshuser is not None:
sshserver = "{}@{}".format(sshuser, sshhostname)
if sshport is not None:
sshserver = "{}:{}".format(sshserver, sshport)
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.connect(
hostname=sshhostname,
port=sshport,
username=sshuser,
key_filename=sshkey,
)
num_jobs = int(num_jobs)
cores_per_job = int(cores_per_job)
start_wait = int(start_wait)
if extra_params is None:
extra_params = {}
max_delay = start_wait * 60
delay = 5 if extra_params.get("run_local") else 15
max_tries = 10
if profile is None:
has_throwaway = True
profile, profile_dir = create_throwaway_profile(
executable, ssh_client, profile_in_work_dir, work_dir
)
print("Created profile {}".format(profile))
sys.stdout.flush()
else:
if ssh_client is None:
if os.path.isdir(profile) and os.path.isabs(profile):
# Return full_path if one is given
profile_dir = profile
else:
profile_dir = locate_profile(profile)
else:
remote_cmd = (
'{0} -E -c '
'"from IPython.utils.path import locate_profile; '
'import os; '
'print(\'{1}\' if os.path.isdir(\'{1}\') and os.path.isabs(\'{1}\') '
'else locate_profile(\'{1}\'))"'
).format(executable, profile)
profile_dir = ssh_client.exec_command(
'source ~/.profile > /dev/null; ' + remote_cmd
)[1].read().decode('utf-8').strip()
# ensure we have an .ipython directory to prevent issues
# creating it during parallel startup
cmd = [executable, "-E", "-c",
"from IPython import start_ipython; start_ipython()",
"profile", "create", "--parallel"] + _get_profile_args(
profile, profile_dir
)
if ssh_client is None:
subprocess.check_call(cmd)
else:
remote_cmd = ' '.join([
'"{}"'.format(item) if ' ' in item else item
for item in cmd
])
ssh_client.exec_command(
'source ~/.profile; ' + remote_cmd
)
has_throwaway = False
num_tries = 0
cluster_id = str(uuid.uuid4())
print("Cluster profile: {}".format(profile))
print("Cluster profile directory: {}".format(profile_dir))
print("Cluster ID: {}".format(cluster_id))
sys.stdout.flush()
while 1:
try:
if extra_params.get("run_local"):
_start_local(num_jobs, profile, profile_dir, cluster_id)
else:
_start(
scheduler, profile, profile_dir, queue, num_jobs,
cores_per_job, cluster_id, extra_params,
executable, ssh_client, cluster=cluster, work_dir=work_dir
)
print("Cluster started.")
sys.stdout.flush()
break
except subprocess.CalledProcessError:
if num_tries > max_tries:
raise
num_tries += 1
time.sleep(delay)
print("Retry to start cluster...")
sys.stdout.flush()
client = None
try:
url_file = get_url_file(
profile, profile_dir, cluster_id, executable,
ssh_client=ssh_client, timeout=start_wait * 60
)
print("URL file: {}".format(url_file))
sys.stdout.flush()
need_engines = 1 # Start using cluster when this many engines are up
client = None
slept = 0
max_up = 0
up = 0
while up < need_engines:
up = _nengines_up(
url_file,
sshserver=sshserver, sshkey=sshkey, sshpassword=sshpassword
)
print("{} engines up.".format(up))
sys.stdout.flush()
if up < max_up:
print(
"Engine(s) that were up have shutdown prematurely. "
"Aborting cluster startup."
)
_stop(profile, profile_dir, cluster_id, executable, ssh_client)
sys.exit(1)
max_up = up
time.sleep(delay)
slept += delay
if slept > max_delay:
raise IOError("Cluster startup timed out.")
client = Client(
url_file, timeout=60,
sshserver=sshserver, sshkey=sshkey, sshpassword=sshpassword
)
if direct:
view = _get_direct_view(client, retries)
else:
view = _get_balanced_blocked_view(client, retries)
view.clusterhelper = {
"profile": profile,
"cluster_id": cluster_id,
"client": client,
}
if dill:
pickleutil.use_dill()
view.apply(pickleutil.use_dill)
yield view
finally:
if client:
_shutdown(client)
_stop(profile, profile_dir, cluster_id, executable, ssh_client)
if has_throwaway:
delete_profile(profile_dir, cluster_id, ssh_client)
ssh_client.close()
def _profile_dir_default(self):
return locate_profile()
from IPython.html.services.config import ConfigManager
from IPython.utils.path import locate_profile
cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))
cm.update('livereveal', {
'theme': 'beige',
'transition': 'fade',
'start_slideshow_at': 'selected',
})
def _profile_dir_default(self):
return locate_profile(self.profile)
def ensure_controller(self, connect_only=False):
"""Make sure a controller is available, else start a local one.
"""
if self._client:
return self._client
if self.profile is None:
self._select_profile()
if self.profile is None:
return None
print "parallelflow: using IPython profile %r" % self.profile
try:
self._client = Client(profile=self.profile)
print "parallelflow: connected to controller"
return self._client
except error.TimeoutError:
print "parallelflow: timeout when connecting to controller"
if connect_only:
start_ctrl = False
elif qt_available:
res = QtGui.QMessageBox.question(
None, "Start controller",
"Unable to connect to the configured IPython "
"controller. Do you want to start one?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
start_ctrl = res == QtGui.QMessageBox.Yes
else:
start_ctrl = True
except IOError:
print "parallelflow: didn't find a controller to connect to"
if connect_only:
start_ctrl = False
elif qt_available:
res = QtGui.QMessageBox.question(
None, "Start controller",
"No controller is configured in this IPython profile. "
"Do you want to start one?",
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
start_ctrl = res == QtGui.QMessageBox.Yes
else:
start_ctrl = True
if start_ctrl:
ctrl_pid = os.path.join(locate_profile(self.profile), 'pid',
'ipcontroller.pid')
if os.path.exists(ctrl_pid):
os.remove(ctrl_pid)
print "parallelflow: starting controller"
proc, code = self.start_process(
lambda: os.path.exists(ctrl_pid), sys.executable, '-m',
'IPython.parallel.apps.ipcontrollerapp',
'--profile=%s' % self.profile)
if code is not None:
if qt_available:
QtGui.QMessageBox.critical(
None, "Error", "Controller exited with code %d" % code)
print(
"parallelflow: controller process exited with "
"code %d" % code)
return None
else:
self.started_controller = proc
print "parallelflow: controller started, connecting"
self._client = Client(profile=self.profile)
return self._client
return None
def get_config_manager(profile):
profile_dir = locate_profile(profile)
return ConfigManager(profile_dir=profile_dir)
