def find_resources(self):
"""This resolves the cluster directory.
This tries to find the cluster directory and if successful, it will
have done:
* Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
the application.
* Sets ``self.cluster_dir`` attribute of the application and config
objects.
The algorithm used for this is as follows:
1. Try ``Global.cluster_dir``.
2. Try using ``Global.profile``.
3. If both of these fail and ``self.auto_create_cluster_dir`` is
``True``, then create the new cluster dir in the IPython directory.
4. If all fails, then raise :class:`ClusterDirError`.
"""
try:
cluster_dir = self.command_line_config.Global.cluster_dir
except AttributeError:
cluster_dir = self.default_config.Global.cluster_dir
cluster_dir = expand_path(cluster_dir)
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
except ClusterDirError:
pass
else:
self.log.info('Using existing cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
return
try:
self.profile = self.command_line_config.Global.profile
except AttributeError:
self.profile = self.default_config.Global.profile
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
self.ipython_dir, self.profile)
except ClusterDirError:
pass
else:
self.log.info('Using existing cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
return
if self.auto_create_cluster_dir:
self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
self.ipython_dir, self.profile
)
self.log.info('Creating new cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
else:
raise ClusterDirError('Could not find a valid cluster directory.')
def find_resources(self):
"""This resolves the cluster directory.
This tries to find the cluster directory and if successful, it will
have done:
* Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
the application.
* Sets ``self.cluster_dir`` attribute of the application and config
objects.
The algorithm used for this is as follows:
1. Try ``Global.cluster_dir``.
2. Try using ``Global.profile``.
3. If both of these fail and ``self.auto_create_cluster_dir`` is
``True``, then create the new cluster dir in the IPython directory.
4. If all fails, then raise :class:`ClusterDirError`.
"""
try:
cluster_dir = self.command_line_config.Global.cluster_dir
except AttributeError:
cluster_dir = self.default_config.Global.cluster_dir
cluster_dir = expand_path(cluster_dir)
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
except ClusterDirError:
pass
else:
self.log.info('Using existing cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
return
try:
self.profile = self.command_line_config.Global.profile
except AttributeError:
self.profile = self.default_config.Global.profile
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
self.ipython_dir, self.profile)
except ClusterDirError:
pass
else:
self.log.info('Using existing cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
return
if self.auto_create_cluster_dir:
self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
self.ipython_dir, self.profile)
self.log.info('Creating new cluster dir: %s' % \
self.cluster_dir_obj.location
)
self.finish_cluster_dir()
else:
raise ClusterDirError('Could not find a valid cluster directory.')
def check_furl_file_security(furl_file, secure):
"""Remove the old furl_file if changing security modes."""
furl_file = expand_path(furl_file)
if os.path.isfile(furl_file):
with open(furl_file, 'r') as f:
oldfurl = f.read().strip()
if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
os.remove(furl_file)
def check_furl_file_security(furl_file, secure):
"""Remove the old furl_file if changing security modes."""
furl_file = expand_path(furl_file)
if os.path.isfile(furl_file):
with open(furl_file, 'r') as f:
oldfurl = f.read().strip()
if (oldfurl.startswith('pb://')
and not secure) or (oldfurl.startswith('pbu://') and secure):
os.remove(furl_file)
def find_profile_dir(cls, profile_dir, config=None):
"""Find/create a profile dir and return its ProfileDir.
This will create the profile directory if it doesn't exist.
Parameters
----------
profile_dir : unicode or str
The path of the profile directory.
"""
profile_dir = expand_path(profile_dir)
if not os.path.isdir(profile_dir):
raise ProfileDirError('Profile directory not found: %s' % profile_dir)
return cls(location=profile_dir, config=config)
def find_profile_dir(cls, profile_dir, config=None):
"""Find/create a profile dir and return its ProfileDir.
This will create the profile directory if it doesn't exist.
Parameters
----------
profile_dir : unicode or str
The path of the profile directory.
"""
profile_dir = expand_path(profile_dir)
if not os.path.isdir(profile_dir):
raise ProfileDirError('Profile directory not found: %s' % profile_dir)
return cls(location=profile_dir, config=config)
def find_furl(furl_or_file):
"""Find, validate and return a FURL in a string or file.
This calls :func:`IPython.utils.path.expand_path` on the argument to
properly handle ``~`` and ``$`` variables in the path.
"""
if is_valid_furl(furl_or_file):
return furl_or_file
furl_or_file = expand_path(furl_or_file)
if is_valid_furl_file(furl_or_file):
with open(furl_or_file, 'r') as f:
furl = f.read().strip()
if is_valid_furl(furl):
return furl
raise FURLError("Not a valid FURL or FURL file: %r" % furl_or_file)
def pre_construct(self):
# The log and security dirs were set earlier, but here we put them
# into the config and log them.
config = self.master_config
sdir = self.cluster_dir_obj.security_dir
self.security_dir = config.Global.security_dir = sdir
ldir = self.cluster_dir_obj.log_dir
self.log_dir = config.Global.log_dir = ldir
pdir = self.cluster_dir_obj.pid_dir
self.pid_dir = config.Global.pid_dir = pdir
self.log.info("Cluster directory set to: %s" % self.cluster_dir)
config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
# Change to the working directory. We do this just before construct
# is called so all the components there have the right working dir.
self.to_work_dir()
def find_furl(furl_or_file):
"""Find, validate and return a FURL in a string or file.
This calls :func:`IPython.utils.path.expand_path` on the argument to
properly handle ``~`` and ``$`` variables in the path.
"""
if is_valid_furl(furl_or_file):
return furl_or_file
furl_or_file = expand_path(furl_or_file)
if is_valid_furl_file(furl_or_file):
with open(furl_or_file, 'r') as f:
furl = f.read().strip()
if is_valid_furl(furl):
return furl
raise FURLError("Not a valid FURL or FURL file: %r" % furl_or_file)
def pre_construct(self):
# The log and security dirs were set earlier, but here we put them
# into the config and log them.
config = self.master_config
sdir = self.cluster_dir_obj.security_dir
self.security_dir = config.Global.security_dir = sdir
ldir = self.cluster_dir_obj.log_dir
self.log_dir = config.Global.log_dir = ldir
pdir = self.cluster_dir_obj.pid_dir
self.pid_dir = config.Global.pid_dir = pdir
self.log.info("Cluster directory set to: %s" % self.cluster_dir)
config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
# Change to the working directory. We do this just before construct
# is called so all the components there have the right working dir.
self.to_work_dir()
def find_cluster_dir(cls, cluster_dir):
"""Find/create a cluster dir and return its ClusterDir.
This will create the cluster directory if it doesn't exist.
Parameters
----------
cluster_dir : unicode or str
The path of the cluster directory. This is expanded using
:func:`IPython.utils.genutils.expand_path`.
"""
cluster_dir = expand_path(cluster_dir)
if not os.path.isdir(cluster_dir):
raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
return ClusterDir(cluster_dir)
def find_cluster_dir(cls, cluster_dir):
"""Find/create a cluster dir and return its ClusterDir.
This will create the cluster directory if it doesn't exist.
Parameters
----------
cluster_dir : unicode or str
The path of the cluster directory. This is expanded using
:func:`IPython.utils.genutils.expand_path`.
"""
cluster_dir = expand_path(cluster_dir)
if not os.path.isdir(cluster_dir):
raise ClusterDirError('Cluster directory not found: %s' %
cluster_dir)
return ClusterDir(location=cluster_dir)
import os
from IPython.utils.path import expand_path
# Configuration file for ipcluster.
c = get_config()
# delay between controller starting and engine starting
c.IPClusterStart.delay = 1.0
# set working directory
work_dir = expand_path('~/ipclusterworkdir')
if not os.path.exists(work_dir):
os.makedirs(work_dir)
c.IPClusterStart.work_dir = work_dir
# set the default number of engines to be 1
c.IPClusterEngines.n = 1
c.IPClusterStart.daemonize = False
c.IPClusterStart.controller_launcher_class = 'LocalControllerLauncher'
c.IPClusterEngines.engine_launcher_class = 'PBSEngineSetLauncher'
c.LocalControllerLauncher.controller_cmd = ['ipcontroller']
c.PBSEngineSetLauncher.job_id_regexp = '\\d+'
c.PBSEngineSetLauncher.submit_command = ['qsub']
c.PBSEngineSetLauncher.delete_command = ['qdel']
c.PBSEngineSetLauncher.queue = u'default'
def render(self, filename, output=None):
"""
Convert the filename to the given output format(s)
"""
# Export each documents
conversion_success = 0
converted_docs = []
# save here to change back after the conversation.
orig_cwd = os.getcwd()
needs_chdir = False
# expand $HOME and so on...
filename = expand_path(filename)
filename = os.path.abspath(filename)
self.log.info("Converting %s..." % filename)
basedir = os.path.dirname(filename)
basename = os.path.splitext(os.path.basename(filename))[0]
# It's easier if we just change wd to the dir of the file
if unicode_type(basedir) != py3compat.getcwd():
os.chdir(basedir)
needs_chdir = True
self.log.info("Changing to working dir: %s" % basedir)
filename = os.path.basename(filename)
outputdir_name = os.path.splitext(basename)[0] + "_files"
# parse the input document
parsed, metadata = self.parse_document(filename)
# get the output formats
# order: kwarg overwrites default overwrites document
output_formats = [self._outputs[self.default_export_format]]
if output is None:
self.log.debug("Converting to default output format [%s]!" %
self.default_export_format)
elif output == "all":
outputs = metadata.get("output", None)
# if nothing is specified, we keep the default
if outputs is None:
self.log.debug(
"Did not find any specified output formats: using only default!"
)
else:
output_formats = []
for fmt_name, config in iteritems(outputs):
fod = self.get_output_format(fmt_name, config)
output_formats.append(fod)
self.log.debug(
"Converting to all specified output formats: %s" %
[fmt.name for fmt in output_formats])
else:
self._ensure_valid_output(output)
output_formats = [self._outputs[output]]
for final_format in output_formats:
self.log.info("Converting document %s to %s", filename,
final_format.name)
# TODO: build a proper way to specify final output...
md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name,
export_config=final_format,
log=self.log,
parent=self)
# get the temporary md file
self.convert(parsed, md_temp)
if final_format.keep_md or self.keep_md:
mdfilename = basename + "." + final_format.name + ".md"
self.log.info("Saving the temporary markdown as '%s'." %
mdfilename)
# TODO: remove the first yaml metadata block and
# put "#<title>\n<author>\n<date>" before the rest
with codecs.open(mdfilename, 'w+b', 'UTF-8') as f:
f.write(md_temp.content)
# convert the md file to the final filetype
input_format = "markdown" \
"+autolink_bare_uris" \
"+ascii_identifiers" \
"+tex_math_single_backslash-implicit_figures" \
"+fenced_code_attributes"
extra = [
"--smart", # typographically correct output (curly quotes, etc)
"--email-obfuscation",
"none", #do not obfuscation email names with javascript
"--self-contained", # include img/scripts as data urls
"--standalone", # html with header + footer
"--section-divs",
]
outfilename = basename + "." + final_format.file_extension
# exported is irrelevant, as we pass in a filename
exported = pandoc(source=md_temp.content,
to=final_format.pandoc_export_format,
format=input_format,
extra_args=extra,
outputfile=outfilename)
self.log.info("Written final output: %s" % outfilename)
converted_docs.append(os.path.join(basedir, outfilename))
if needs_chdir:
os.chdir(orig_cwd)
return converted_docs
def _work_dir_changed(self, name, old, new):
self.work_dir = unicode_type(expand_path(new))
def _work_dir_changed(self, name, old, new):
self.work_dir = str(expand_path(new))
def render(self, filename, output=None):
"""
Convert the filename to the given output format(s)
"""
# Export each documents
conversion_success = 0
converted_docs = []
# save here to change back after the conversation.
orig_cwd = os.getcwd()
needs_chdir = False
# expand $HOME and so on...
filename = expand_path(filename)
filename = os.path.abspath(filename)
self.log.info("Converting %s..." % filename)
basedir = os.path.dirname(filename)
basename = os.path.splitext(os.path.basename(filename))[0]
# It's easier if we just change wd to the dir of the file
if unicode_type(basedir) != py3compat.getcwd():
os.chdir(basedir)
needs_chdir = True
self.log.info("Changing to working dir: %s" % basedir)
filename = os.path.basename(filename)
outputdir_name = os.path.splitext(basename)[0] + "_files"
# parse the input document
parsed, metadata = self.parse_document(filename)
# get the output formats
# order: kwarg overwrites default overwrites document
output_formats = [self._outputs[self.default_export_format]]
if output is None:
self.log.debug("Converting to default output format [%s]!" % self.default_export_format)
elif output == "all":
outputs = metadata.get("output", None)
# if nothing is specified, we keep the default
if outputs is None:
self.log.debug("Did not find any specified output formats: using only default!")
else:
output_formats = []
for fmt_name, config in iteritems(outputs):
fod = self.get_output_format(fmt_name, config)
output_formats.append(fod)
self.log.debug("Converting to all specified output formats: %s" %
[fmt.name for fmt in output_formats])
else:
self._ensure_valid_output(output)
output_formats = [self._outputs[output]]
for final_format in output_formats:
self.log.info("Converting document %s to %s", filename, final_format.name)
# TODO: build a proper way to specify final output...
md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name,
export_config=final_format,
log=self.log, parent=self)
# get the temporary md file
self.convert(parsed, md_temp)
if final_format.keep_md or self.keep_md:
mdfilename = basename+"."+final_format.name+".md"
self.log.info("Saving the temporary markdown as '%s'." % mdfilename)
# TODO: remove the first yaml metadata block and
# put "#<title>\n<author>\n<date>" before the rest
with codecs.open(mdfilename, 'w+b','UTF-8') as f:
f.write(md_temp.content)
# convert the md file to the final filetype
input_format = "markdown" \
"+autolink_bare_uris" \
"+ascii_identifiers" \
"+tex_math_single_backslash-implicit_figures" \
"+fenced_code_attributes"
extra = ["--smart", # typographically correct output (curly quotes, etc)
"--email-obfuscation", "none", #do not obfuscation email names with javascript
"--self-contained", # include img/scripts as data urls
"--standalone", # html with header + footer
"--section-divs",
]
outfilename = basename+"." +final_format.file_extension
# exported is irrelevant, as we pass in a filename
exported = pandoc(source=md_temp.content,
to=final_format.pandoc_export_format,
format=input_format,
extra_args=extra,
outputfile=outfilename)
self.log.info("Written final output: %s" % outfilename)
converted_docs.append(os.path.join(basedir, outfilename))
if needs_chdir:
os.chdir(orig_cwd)
return converted_docs
def _work_dir_changed(self, change):
self.work_dir = unicode_type(expand_path(change['new']))
def _work_dir_changed(self, name, old, new):
self.work_dir = unicode(expand_path(new))
def _work_dir_changed(self, change):
self.work_dir = str(expand_path(change['new']))
