def load_export_config(export_config_path):
"""load the export configuration"""
if isinstance(export_config_path, string_types):
export_config_path = pathlib.Path(export_config_path)
data = read_file_from_directory(
export_config_path.parent,
export_config_path.name,
"export configuration",
logger,
interp_ext=True,
)
# validate against schema
global _EXPORT_SCHEMA
if _EXPORT_SCHEMA is None:
# lazy load schema once
_EXPORT_SCHEMA = read_file_from_directory(
get_module_path(schema),
_EXPORT_SCHEMA_FILE,
"export configuration schema",
logger,
interp_ext=True,
)
try:
jsonschema.validate(data, _EXPORT_SCHEMA)
except jsonschema.ValidationError as err:
handle_error(
"validation of export config {} failed against {}: {}".format(
export_config_path, _EXPORT_SCHEMA_FILE, err.message),
jsonschema.ValidationError,
logger=logger,
)
return data
def make_conf(overwrite=False, **kwargs):
""" create the contents of a sphinx config.py
with default values compliant with ipypublish.
Parameters
----------
overwrite: bool
if True, default arguments are overwritten by key-word arguments
kwargs:
key-word arguments are included as <key> = <val>
note docstring is a special key, inserted at the top of the file
Returns
-------
conf_content: str
"""
# load local config.yaml
path = os.path.join(os.path.dirname(os.path.realpath(__file__)))
data = read_file_from_directory(path,
"config.yaml",
"sphinx config",
logger,
interp_ext=True)
conf_str = []
docstring = None
if "docstring" in data:
docstring = data.pop("docstring")
if "docstring" in kwargs:
kwdocstring = data.pop("docstring")
if overwrite or not docstring:
docstring = kwdocstring
if docstring:
conf_str.append('"""')
conf_str.append(str(docstring))
conf_str.append('"""')
for key, val in data.items():
if key in kwargs and overwrite:
continue
conf_str.append("")
if isinstance(val, string_types):
val = '"{}"'.format(val)
conf_str.append("{0} = {1}".format(key, val))
for key, val in kwargs.items():
if key in data and not overwrite:
continue
conf_str.append("")
if isinstance(val, string_types):
val = '"{}"'.format(val)
conf_str.append("{0} = {1}".format(key, val))
return "\n".join(conf_str) + "\n"
def load_template(template_key, template_dict):
if template_dict is None:
return None
if "directory" in template_dict["outline"]:
outline_template = read_file_from_directory(
template_dict["outline"]["directory"],
template_dict["outline"]["file"],
"template outline", logger, interp_ext=False)
outline_name = os.path.join(template_dict["outline"]["directory"],
template_dict["outline"]["file"])
else:
outline_template = read_file_from_module(
template_dict["outline"]["module"],
template_dict["outline"]["file"],
"template outline", logger, interp_ext=False)
outline_name = os.path.join(template_dict["outline"]["module"],
template_dict["outline"]["file"])
segments = []
for snum, segment in enumerate(template_dict.get("segments", [])):
if "file" not in segment:
handle_error(
"'file' expected in segment {}".format(snum),
KeyError, logger)
if "directory" in segment:
seg_data = read_file_from_directory(
segment["directory"],
segment["file"], "template segment", logger, interp_ext=True)
elif "module" in segment:
seg_data = read_file_from_module(
segment["module"],
segment["file"], "template segment", logger, interp_ext=True)
else:
handle_error(
"'directory' or 'module' expected in segment {}".format(snum),
KeyError, logger)
segments.append(seg_data)
template_str = create_template(outline_template, outline_name, segments)
return str_to_jinja(template_str, template_key)
def create_template(outline_template,
outline_name,
segment_datas,
outpath=None):
# type: (dict, Tuple[dict]) -> str
""" build a latex jinja template from;
- a jinja(2) template outline,
which may contain segment placeholders,
- and json segment files adhering to the segment.schema.json schema
if a segment contains the key "overwrite",
then its value should be a list of keys,
such that these key values overwrite any entries before
Parameters
----------
outline_template: str
segment_datas: tuple or dict
outpath: None or str
if not None, output to path
"""
# get the placeholders @ipubreplace{above|below}{name}
regex = re.compile("\\@ipubreplace\\{([^\\}]+)\\}\\{([^\\}]+)\\}",
re.MULTILINE)
placeholder_tuple = regex.findall(outline_template)
if not placeholder_tuple:
if segment_datas:
handle_error(
"the segment data is provided, " +
"but the outline template contains no placeholders",
KeyError,
logger,
)
if outpath:
_output_to_file(outline_template, outpath)
return outline_template
placeholders = {name: append for append, name in placeholder_tuple}
# TODO validate that placeholders to not exist multiple times,
# with above and below
replacements = {key: "" for key in placeholders.keys()}
docstrings = ["outline: {}".format(outline_name)]
if segment_datas:
docstrings.append("with segments:")
global _SEGMENT_SCHEMA
if _SEGMENT_SCHEMA is None:
# lazy segment schema once
_SEGMENT_SCHEMA = read_file_from_directory(
get_module_path(schema),
_SEGMENT_SCHEMA_FILE,
"segment configuration schema",
logger,
interp_ext=True,
)
for seg_num, segment_data in enumerate(segment_datas):
# validate segment
try:
jsonschema.validate(segment_data, _SEGMENT_SCHEMA)
except jsonschema.ValidationError as err:
handle_error(
"validation of template segment {} failed: {}".format(
seg_num, err.message),
jsonschema.ValidationError,
logger=logger,
)
# get description of segment
docstrings.append("- {0}: {1}".format(segment_data["identifier"],
segment_data["description"]))
# find what key to overwrite
overwrite = segment_data.get("overwrite", [])
logger.debug("overwrite keys: {}".format(overwrite))
for key, segtext in segment_data.get("segments").items():
if key not in placeholders:
handle_error(
"the segment key '{}' ".format(key) +
"is not contained in the outline template",
KeyError,
logger,
)
if not isinstance(segtext, string_types):
segtext = "\n".join(segtext)
if key in overwrite:
replacements[key] = segtext
elif placeholders[key] == "above":
replacements[key] = segtext + "\n" + replacements[key]
elif placeholders[key] == "below":
replacements[key] = replacements[key] + "\n" + segtext
else:
handle_error(
("the placeholder @ipubreplace{{{0}}}{{{1}}} ".format(
key, placeholders[key]) +
"should specify 'above' or 'below' appending"),
jsonschema.ValidationError,
logger=logger,
)
if "meta_docstring" in placeholders:
docstring = "\n".join([s for s in docstrings if s]).replace("'", '"')
replacements["meta_docstring"] = docstring
if "ipypub_version" in placeholders:
# TODO add option to include ipypub version in output file
# not included by default,
# since tests need to be changed to ignore version number
replacements["ipypub_version"] = "" # str(__version__)
prefix = "@ipubreplace{"
replace_dict = {
prefix + append + "}{" + name + "}": replacements.get(name, "")
for append, name in placeholder_tuple
}
outline = multireplace(outline_template, replace_dict)
if outpath:
_output_to_file(outline, outpath)
return outline
def publish(self, ipynb_path, nb_node=None):
""" convert one or more Jupyter notebooks to a published format
paths can be string of an existing file or folder,
or a pathlib.Path like object
all files linked in the documents are placed into a single files_folder
Parameters
----------
ipynb_path: str or pathlib.Path
notebook file or directory
nb_node: None or nbformat.NotebookNode
a pre-converted notebook
Returns
--------
outdata: dict
containing keys;
"outpath", "exporter", "stream", "main_filepath", "resources"
"""
# setup the input and output paths
if isinstance(ipynb_path, string_types):
ipynb_path = pathlib.Path(ipynb_path)
ipynb_name, ipynb_ext = os.path.splitext(ipynb_path.name)
outdir = (os.path.join(os.getcwd(), "converted")
if self.outpath is None else str(self.outpath))
with self._log_handlers(ipynb_name, outdir):
if not ipynb_path.exists() and not nb_node:
handle_error(
"the notebook path does not exist: {}".format(ipynb_path),
IOError,
self.logger,
)
# log start of conversion
self.logger.info("started ipypublish v{0} at {1}".format(
ipypublish.__version__, time.strftime("%c")))
self.logger.info("logging to: {}".format(
os.path.join(outdir, ipynb_name + ".nbpub.log")))
self.logger.info("running for ipynb(s) at: {0}".format(ipynb_path))
self.logger.info("with conversion configuration: {0}".format(
self.conversion))
if nb_node is None and ipynb_ext in self.pre_conversion_funcs:
func = self.pre_conversion_funcs[ipynb_ext]
self.logger.info("running pre-conversion with: {}".format(
inspect.getmodule(func)))
try:
nb_node = func(ipynb_path)
except Exception as err:
handle_error(
"pre-conversion failed for {}: {}".format(
ipynb_path, err),
err,
self.logger,
)
# doesn't work with folders
# if (ipynb_ext != ".ipynb" and nb_node is None):
# handle_error(
# 'the file extension is not associated with any '
# 'pre-converter: {}'.format(ipynb_ext),
# TypeError, self.logger)
if nb_node is None:
# merge all notebooks
# TODO allow notebooks to remain separate
# (would require creating a main.tex with the preamble in etc )
# Could make everything a 'PyProcess',
# with support for multiple streams
final_nb, meta_path = merge_notebooks(
ipynb_path, ignore_prefix=self.ignore_prefix)
else:
final_nb, meta_path = (nb_node, ipynb_path)
# validate the notebook metadata against the schema
if self.validate_nb_metadata:
nb_metadata_schema = read_file_from_directory(
get_module_path(schema),
"doc_metadata.schema.json",
"doc_metadata.schema",
self.logger,
interp_ext=True,
)
try:
jsonschema.validate(final_nb.metadata, nb_metadata_schema)
except jsonschema.ValidationError as err:
handle_error(
"validation of notebook level metadata failed: {}\n"
"see the doc_metadata.schema.json for full spec".
format(err.message),
jsonschema.ValidationError,
logger=self.logger,
)
# set text replacements for export configuration
replacements = {
self.meta_path_placeholder:
str(meta_path),
self.files_folder_placeholder:
"{}{}".format(get_valid_filename(ipynb_name),
self.folder_suffix),
}
self.logger.debug("notebooks meta path: {}".format(meta_path))
# load configuration file
(
exporter_cls,
jinja_template,
econfig,
pprocs,
pconfig,
) = self._load_config_file(replacements)
# run nbconvert
self.logger.info("running nbconvert")
exporter, stream, resources = self.export_notebook(
final_nb, exporter_cls, econfig, jinja_template)
# postprocess results
main_filepath = os.path.join(outdir,
ipynb_name + exporter.file_extension)
for post_proc_name in pprocs:
proc_class = find_entry_point(
post_proc_name,
"ipypublish.postprocessors",
self.logger,
"ipypublish",
)
proc = proc_class(pconfig)
stream, main_filepath, resources = proc.postprocess(
stream, exporter.output_mimetype, main_filepath, resources)
self.logger.info("process finished successfully")
return {
"outpath": outdir,
"exporter": exporter,
"stream": stream,
"main_filepath": main_filepath,
"resources": resources,
}
