def export_to_scala(absolute_notebook_path):
'''convert the notebook source to scala'''
exporter = TemplateExporter(extra_loaders=[FileSystemLoader(INSTALLDIR)],
preprocessors=[ScalaAppPreprocessor])
exporter.template_file = 'scala_sparkapp'
(body, resources) = exporter.from_file(absolute_notebook_path)
return body
def export_to_scala(absolute_notebook_path):
'''convert the notebook source to scala'''
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
exporter = TemplateExporter(extra_loaders=[FileSystemLoader(INSTALLDIR)],
preprocessors=[ScalaAppPreprocessor])
exporter.template_file = 'scala_sparkapp'
(body, resources) = exporter.from_file(absolute_notebook_path)
return body
def __init__(self, template_file, strip_outputs=True,
write_outputs=False, output_dir='./figures'):
"""template_file - location of jinja template to use for export
strip_outputs - whether to remove output cells from the output
"""
filters = [
('string2json', self.string2json),
('create_input_codeblock', self.create_input_codeblock),
('create_output_codeblock', self.create_output_codeblock),
('create_output_block', self.create_output_block),
('create_attributes', self.create_attributes),
('dequote', self.dequote),
('data2uri', self.data2uri)
]
import jinja2
# need to create a jinja loader that looks in whatever
# arbitrary path we have passed in for the template_file
direct_loader = jinja2.FileSystemLoader(os.path.dirname(template_file))
self.exporter = TemplateExporter(extra_loaders=[direct_loader])
self.exporter.output_mimetype = 'text/markdown'
self.exporter.file_extension = '.md'
# have to register filters before setting template file for
# ipython 3 compatibility
for name, filter in filters:
self.exporter.register_filter(name, filter)
self.exporter.template_file = os.path.basename(template_file)
logging.debug("Creating MarkdownWriter")
logging.debug(("MarkdownWriter: template_file = %s"
% template_file))
logging.debug(("MarkdownWriter.exporter.template_file = %s"
% self.exporter.template_file))
logging.debug(("MarkdownWriter.exporter.filters = %s"
% self.exporter.environment.filters.keys()))
self.strip_outputs = strip_outputs
self.write_outputs = write_outputs
self.output_dir = output_dir
class MarkdownWriter(NotebookWriter):
"""Write a notebook into markdown."""
def __init__(self, template_file, strip_outputs=True,
write_outputs=False, output_dir='./figures'):
"""template_file - location of jinja template to use for export
strip_outputs - whether to remove output cells from the output
"""
filters = [
('string2json', self.string2json),
('create_input_codeblock', self.create_input_codeblock),
('create_output_codeblock', self.create_output_codeblock),
('create_output_block', self.create_output_block),
('create_attributes', self.create_attributes),
('dequote', self.dequote),
('data2uri', self.data2uri)
]
import jinja2
# need to create a jinja loader that looks in whatever
# arbitrary path we have passed in for the template_file
direct_loader = jinja2.FileSystemLoader(os.path.dirname(template_file))
self.exporter = TemplateExporter(extra_loaders=[direct_loader])
self.exporter.output_mimetype = 'text/markdown'
self.exporter.file_extension = '.md'
# have to register filters before setting template file for
# ipython 3 compatibility
for name, filter in filters:
self.exporter.register_filter(name, filter)
self.exporter.template_file = os.path.basename(template_file)
logging.debug("Creating MarkdownWriter")
logging.debug(("MarkdownWriter: template_file = %s"
% template_file))
logging.debug(("MarkdownWriter.exporter.template_file = %s"
% self.exporter.template_file))
logging.debug(("MarkdownWriter.exporter.filters = %s"
% self.exporter.environment.filters.keys()))
self.strip_outputs = strip_outputs
self.write_outputs = write_outputs
self.output_dir = output_dir
def write_from_json(self, notebook_json):
notebook = v4.reads_json(notebook_json)
return self.write(notebook)
def writes(self, notebook):
body, resources = self.exporter.from_notebook_node(notebook)
self.resources = resources
if self.write_outputs:
self.write_resources(resources)
# remove any blank lines added at start and end by template
text = re.sub(r'\A\s*\n|^\s*\Z', '', body)
return cast_unicode(text, 'utf-8')
def write_resources(self, resources):
"""Write the output data in resources returned by exporter
to files.
"""
for filename, data in list(resources.get('outputs', {}).items()):
# Determine where to write the file to
dest = os.path.join(self.output_dir, filename)
path = os.path.dirname(dest)
if path and not os.path.isdir(path):
os.makedirs(path)
# Write file
with open(dest, 'wb') as f:
f.write(data)
# --- filter functions to be used in the output template --- #
def string2json(self, string):
"""Convert json into its string representation.
Used for writing outputs to markdown."""
kwargs = {
'cls': BytesEncoder, # use the IPython bytes encoder
'indent': 1,
'sort_keys': True,
'separators': (',', ': '),
}
return cast_unicode(json.dumps(string, **kwargs), 'utf-8')
def create_input_codeblock(self, cell):
codeblock = cast_unicode('{fence}{attributes}\n'
'{cell.source}\n'
'{fence}')
attrs = self.create_attributes(cell, cell_type='input')
return codeblock.format(attributes=attrs, fence='```', cell=cell)
def create_output_block(self, cell):
if self.strip_outputs:
return ''
else:
return self.create_output_codeblock(cell)
def create_output_codeblock(self, cell):
codeblock = ('{fence}{{.json .output n={execution_count}}}\n'
'{contents}\n'
'{fence}')
return codeblock.format(fence='```',
execution_count=cell.execution_count,
contents=self.string2json(cell.outputs))
def create_attributes(self, cell, cell_type=None):
"""Turn the attribute dict into an attribute string
for the code block.
"""
if self.strip_outputs or not hasattr(cell, 'execution_count'):
return 'python'
attrs = cell.metadata.get('attributes')
attr = PandocAttributes(attrs, 'dict')
if 'python' in attr.classes:
attr.classes.remove('python')
if 'input' in attr.classes:
attr.classes.remove('input')
if cell_type == 'figure':
attr.kvs.pop('caption', '')
attr.classes.append('figure')
attr.classes.append('output')
return attr.to_html()
elif cell_type == 'input':
# ensure python goes first so that github highlights it
attr.classes.insert(0, 'python')
attr.classes.insert(1, 'input')
if cell.execution_count:
attr.kvs['n'] = cell.execution_count
return attr.to_markdown(format='{classes} {id} {kvs}')
else:
return attr.to_markdown()
@staticmethod
def dequote(s):
"""Remove excess quotes from a string."""
if len(s) < 2:
return s
elif (s[0] == s[-1]) and s.startswith(('"', "'")):
return s[1: -1]
else:
return s
@staticmethod
def data2uri(data, data_type):
"""Convert base64 data into a data uri with the given data_type."""
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
'image/svg+xml': 'svg',
}
inverse_map = {v: k for k, v in list(MIME_MAP.items())}
mime_type = inverse_map[data_type]
uri = r"data:{mime};base64,{data}"
return uri.format(mime=mime_type,
data=data[mime_type].replace('\n', ''))