def ipython():
'''Configuration for IPython
'''
# Line which must be appended to IPython config
config_line = '''c.InteractiveShellApp.extensions = ['pdflatex_magic', 'bash2_magic']'''
# Location of IPython default profile
try:
profile = locate_profile()
except (ProfileDirError, OSError):
start_ipython(argv=['profile', 'create', 'default'])
profile = locate_profile()
# Append extensions line to end of profile config
config_file = os.path.join(profile, 'ipython_config.py')
# Open config file
fh = open(config_file, 'r+')
# Search for line to be configured
# Ensure it isn't a comment
line_list = []
for line in fh.readlines():
if 'c.InteractiveShellApp.extensions' in line:
if not line.lstrip().startswith('#'):
line_list.append(line)
# There should be at most one, othewise config is broken
# If config is broken, this won't break it more :-P
if len(line_list) == 1:
# Extract current extensions
before = line_list[0].split('[')
after = before[1].split(']')
# If none, just put new ones straight it
if after[0].strip() == '':
after[0] = "'pdflatex_magic', 'bash2_magic'"
else:
if "'pdflatex_magic'" not in after[0]:
after[0] += ", 'pdflatex_magic'"
if "'bash2_magic'" not in after[0]:
after[0] += ", 'bash2_magic'"
config_line = before[0] + '[' + ']'.join(after)
fh.close()
# To substitute into existing file we need to create a copy
replace(config_file, line_list[0], config_line)
else:
# Write config line to file (at the end)
fh.writelines(config_line)
fh.close()
def _create_base_ipython_dirs():
"""Create default user directories to prevent potential race conditions downstream.
"""
utils.safe_makedir(get_ipython_dir())
ProfileDir.create_profile_dir_by_name(get_ipython_dir())
utils.safe_makedir(os.path.join(get_ipython_dir(), "db"))
utils.safe_makedir(os.path.join(locate_profile(), "db"))
def patch():
from notebook.services.config import ConfigManager
from IPython.paths import locate_profile
cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))
cm.update('livereveal', {'width': '100%', 'height': 700, 'margin': 0.2,})
from IPython.display import HTML
HTML('''<style>.CodeMirror{min-width:100% !important;}</style>''')
def test_sys_path_profile_dir():
"""test that sys.path doesn't get messed up when `--profile-dir` is specified"""
with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
stdout, stderr = assemble_output(kc.iopub_channel)
nt.assert_equal(stdout, "''\n")
def locate_profile(profile='default'):
warn(
"locate_profile has moved to the IPython.paths module since IPython 4.0.",
DeprecationWarning,
stacklevel=2)
from IPython.paths import locate_profile
return locate_profile(profile=profile)
def setup_ipython_logger():
"""Plug and play logging. No params so you can import and forget.
Uses the default ``datefmt`` that comes with :class:`~logging.Formatter`
classes. For example::
>>> default_time_format = '%Y-%m-%d %H:%M:%S'
Can be found under the ``__init__()`` method of the logging
:class:`~logging.Formatter`.
"""
from IPython.paths import locate_profile
ipython_profile = Path(locate_profile())
log_title = "_log-" + time.strftime("%Y-%m-%d")
log_name = ipython_profile.joinpath("log", "_log-" + log_title + ".log")
logger = logging.getLogger(name=log_title)
logger.setLevel(logging.WARNING)
file_handler = logging.FileHandler(log_name, encoding="utf-8")
formatter = logging.Formatter(
"%(asctime)s : %(levelname)s : %(module)s : %(name)s : %(message)s")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
def create_pbs_profile(profile='pbs', local_controller=False):
try:
shutil.rmtree(os.path.expanduser(f'~/.ipython/profile_{profile}'))
except:
pass
create_parallel_profile(profile)
ipcluster = [
"c.IPClusterEngines.engine_launcher_class = 'PBSEngineSetLauncher'"
]
if not local_controller:
ipcluster.append(
"c.IPClusterStart.controller_launcher_class = 'PBSControllerLauncher'"
)
f = {
'ipcluster_config.py':
ipcluster,
'ipcontroller_config.py':
["c.HubFactory.ip = u'*'", "c.HubFactory.registration_timeout = 600"],
'ipengine_config.py': [
"c.IPEngineApp.wait_for_url_file = 300",
"c.EngineFactory.timeout = 300"
]
}
for fname, line in f.items():
fname = os.path.join(locate_profile(profile), fname)
line_prepender(fname, line)
print(f'Succesfully created a new {profile} profile.')
def test_sys_path_profile_dir():
"""test that sys.path doesn't get messed up when `--profile-dir` is specified"""
with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
stdout, stderr = assemble_output(kc.iopub_channel)
nt.assert_equal(stdout, "''\n")
def locate_profile(profile="default"):
warn(
"locate_profile has moved to the IPython.paths module since IPython 4.0.",
stacklevel=2,
)
from IPython.paths import locate_profile
return locate_profile(profile=profile)
def get_url_file(profile, cluster_id):
url_file = "ipcontroller-{0}-client.json".format(cluster_id)
if os.path.isdir(profile) and os.path.isabs(profile):
# Return full_path if one is given
return os.path.join(profile, "security", url_file)
return os.path.join(locate_profile(profile), "security", url_file)
def test_sys_path_profile_dir():
"""test that sys.path doesn't get messed up when `--profile-dir` is specified"""
with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
msg_id, content = execute(kc=kc, code="import sys; print(repr(sys.path))")
stdout, stderr = assemble_output(kc.iopub_channel)
# for error-output on failure
sys.stderr.write(stderr)
sys_path = ast.literal_eval(stdout.strip())
assert '' in sys_path
def _get_hist_file_name(self, profile='default'):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return os.path.join(locate_profile(profile), 'history.sqlite')
def _get_hist_file_name(self, profile='default'):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return Path(locate_profile(profile)) / "history.sqlite"
def pidFileExists(profile, clusterId):
"""
Check if a process-id (pid) file exists for this profile and clusterId.
:param profile: (str) profile name
:param clusterId: (str) cluster ID
:return: (bool) True if computed pid file pathname exists, else False
"""
profileDir = locate_profile(profile)
basename = 'ipcluster-%s.pid' % clusterId
pidFile = os.path.join(profileDir, 'pid', basename)
return os.path.isfile(pidFile)
def __enter__(self):
if JUPYTER:
self.pdir = jupyter_config_dir()
self.cff_name = 'jupyter_notebook_config.json'
else:
self.pdir = locate_profile(self.profile)
self.cff_name = 'ipython_notebook_config.json'
jc = JSONFileConfigLoader(self.cff_name, self.pdir)
try:
self.config = jc.load_config();
except (ConfigFileNotFound,ValueError):
self.config = Config()
return self.config
def __enter__(self):
if JUPYTER:
self.pdir = jupyter_config_dir()
self.cff_name = 'jupyter_notebook_config.json'
else:
self.pdir = locate_profile(self.profile)
self.cff_name = 'ipython_notebook_config.json'
jc = JSONFileConfigLoader(self.cff_name, self.pdir)
try:
self.config = jc.load_config();
except (ConfigFileNotFound,ValueError):
self.config = Config()
return self.config
def get_config():
ip = get_ipython()
if ip is None:
profile_dir = paths.locate_profile()
else:
profile_dir = ip.profile_dir.location
json_path = path.join(profile_dir, "ipython_config.json")
try:
with open(json_path, 'r') as f:
config = json.load(f)
except (FileNotFoundError, JSONDecodeError):
config = {}
return config, json_path
def get_metadata_engine(other_engine):
"""Create and return an SA engine for which will be used for
storing ipydb db metadata about the input engine.
Args:
other_engine - SA engine for which we will be storing metadata for.
Returns:
tuple (dbname, sa_engine). dbname is a unique key for the input
other_engine. sa_engine is the SA engine that will be used for storing
metadata about `other_engine`
"""
path = os.path.join(locate_profile(), 'ipydb')
if not os.path.exists(path):
os.makedirs(path)
dbfilename = get_db_filename(other_engine)
dburl = u'sqlite:////%s' % os.path.join(path, dbfilename)
return dbfilename, sa.create_engine(dburl)
def delete_profile(profile):
MAX_TRIES = 10
dir_to_remove = locate_profile(profile)
if os.path.exists(dir_to_remove):
num_tries = 0
while True:
try:
shutil.rmtree(dir_to_remove)
break
except OSError:
if num_tries > MAX_TRIES:
raise
time.sleep(5)
num_tries += 1
else:
raise ValueError("Cannot find {0} to remove, "
"something is wrong.".format(dir_to_remove))
def templatePath(scheduler, profile, clusterId, process):
profileDir = locate_profile(profile)
basename = '%s_%s_%s.template' % (scheduler, clusterId, process)
filename = os.path.join(profileDir, basename)
return filename
def add_lines_in_profile(profile, files_lines_dict):
for fname, line in files_lines_dict.items():
fname = os.path.join(locate_profile(profile), fname)
line_prepender(fname, line)
def add_lines_in_profile(profile, files_lines_dict):
for fname, line in files_lines_dict.items():
fname = os.path.join(locate_profile(profile), fname)
line_prepender(fname, line)
def locate_profile(profile='default'):
warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
from IPython.paths import locate_profile
return locate_profile(profile=profile)
import sys
from IPython.terminal import interactiveshell
from IPython.paths import locate_profile
sys.path.insert(1, locate_profile())
from _solarized import SolarizedStyle, overrides
def get_style_by_name(name, original=interactiveshell.get_style_by_name):
return SolarizedStyle if name == 'solarized' else original(name)
interactiveshell.get_style_by_name = get_style_by_name
c.TerminalInteractiveShell.highlighting_style = 'solarized'
c.TerminalInteractiveShell.highlighting_style_overrides = overrides
def _saveBatchFiles(numTrials, argDict):
"""
Create and save engine and controller batch files that will
be used by ipcontroller and ipengine at start-up. The files are
created in the profile directory, indicated in `argDict` either
as "profile_dir" or constructed from "profile" by ipython's
`locate_profile` function. The ipyparallel config files must set
the batch_template_file to the names of the generated files, which
are "{scheduler}-{cluster_id}-{process}.template",
where {process} is "engine" or "controller", depending as appropriate.
So for SLURM (the default scheduler), and the default cluster id ("mcs"),
the two generated files are "slurm-mcs-engine.template" and
"slurm-mcs-controller.template".
:param numTrials: (int) the number of GCAM trials to run
:param argDict: (dict) values for "scheduler", "account", "queue",
"profile", "cluster_id", "tasks_per_node", "num_engines", "timelimit",
and "minimum_seconds" are substituted into the batch templates. For
most of these, defaults are taken from ~/.pygcam.cfg and overridden
by argDict. Both "ntasks" and "timelimit" are computed on-the-fly.
:return: (dict) the names of the generated files, keyed by 'engine' or 'controller'
"""
import pipes
minutesPerRun = argDict['minutesPerRun']
maxEngines = argDict['maxEngines']
numEngines = min(numTrials, maxEngines)
maxRunsPerEngine = numTrials // numEngines + (1 if numTrials %
numEngines else 0)
minutesPerEngine = minutesPerRun * maxRunsPerEngine
timelimit = "%02d:%02d:00" % (minutesPerEngine // 60,
minutesPerEngine % 60)
defaults = {
'scheduler': getParam('IPP.Scheduler'),
'account': getParam('IPP.Account'),
'queue': getParam('IPP.Queue'),
'engine_args': getParam('IPP.OtherEngineArgs'),
'cluster_id': argDict['clusterId'],
'tasks_per_node': getParamAsInt('IPP.TasksPerNode'),
'min_secs_to_run': getParamAsInt('IPP.MinTimeToRun') * 60,
'timelimit': timelimit,
'prolog_script': getParam('IPP.PrologScript'),
'epilog_script': getParam('IPP.EpilogScript'),
}
defaults.update(argDict)
profile = defaults['profile']
scheduler = defaults['scheduler']
cluster_id = defaults['cluster_id']
defaults['profile_dir'] = locate_profile(profile)
templates = batchTemplates[scheduler.lower()]
files = {}
# N.B. process is 'controller' or 'engine'
for process, template in templates.items():
cmd_argv = [sys.executable, "-m", "ipyparallel.%s" % process]
text = template % ' '.join(map(pipes.quote,
cmd_argv)) # insert command line
text = text.format(**defaults) # insert other parameters
files[process] = filename = templatePath(scheduler, profile,
cluster_id, process)
_logger.debug('Writing %s', filename)
with open(filename, 'w') as f:
f.write(text)
os.chmod(filename,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) # make executable
return files
count = len(possible_modules)
for module in possible_modules:
try:
return __import__(module, fromlist=[needed_module])
except ImportError:
count -= 1
if count == 0:
raise
try:
# IPython 5.0 and newer
from IPython.terminal.debugger import TerminalPdb
from IPython.core.debugger import BdbQuit_excepthook
from IPython.paths import locate_profile
try:
history_path = os.path.join(locate_profile(), 'ipdb_history')
except (IOError, OSError):
history_path = os.path.join(os.path.expanduser('~'), '.ipdb_history')
class Pdb(TerminalPdb):
def __init__(self, *args, **kwargs):
"""Init pdb and load the history file if present"""
super(Pdb, self).__init__(*args, **kwargs)
try:
with open(history_path, 'r') as f:
self.shell.debugger_history.strings = [
unicode(line.replace(os.linesep, ''))
for line in f.readlines()
]
except IOError:
pass
def locate_profile(profile='default'):
warn("locate_profile has moved to the IPython.paths module")
from IPython.paths import locate_profile
return locate_profile(profile=profile)
def locate_profile(profile='default'):
warn("locate_profile has moved to the IPython.paths module")
from IPython.paths import locate_profile
return locate_profile(profile=profile)
try:
return __import__(module, fromlist=[needed_module])
except ImportError:
count -= 1
if count == 0:
raise
try:
# IPython 5.0 and newer
from IPython.terminal.debugger import TerminalPdb
from IPython.core.debugger import BdbQuit_excepthook
from IPython.paths import locate_profile
try:
history_path = os.path.join(locate_profile(), 'ipdb_history')
except (IOError, OSError):
history_path = os.path.join(os.path.expanduser('~'), '.ipdb_history')
class Pdb(TerminalPdb):
def __init__(self, *args, **kwargs):
"""Init pdb and load the history file if present"""
super(Pdb, self).__init__(*args, **kwargs)
try:
with open(history_path, 'r') as f:
for line in f.readlines():
line = str(line.replace(os.linesep, ''))
self.shell.debugger_history._loaded_strings.append(
line)
except IOError:
open(history_path, 'a')