def run(docs,
variables=None,
inputs_dir='.',
outputs_dir='.',
imports_dir='.'):
for basename, doc in docs.items():
languages = doc.get('languages', 'py java go').split()
for lang in languages:
ipynb_file = '/'.join(
[outputs_dir, '{}-{}.ipynb'.format(basename, lang)])
notebook = md2ipynb.new_notebook(
input_file=os.path.join(inputs_dir, basename + '.md'),
variables=variables,
imports={
i: [os.path.join(imports_dir, path) for path in imports]
for i, imports in doc.get('imports', {}).items()
},
notebook_title=doc.get(
'title',
os.path.basename(basename).replace('-', ' ')),
keep_classes=['language-' + lang, 'shell-sh'],
docs_url='https://beam.apache.org/' +
basename.replace('-', ''),
docs_logo_url=docs_logo_url,
github_ipynb_url='https://github.com/apache/beam/blob/master/'
+ ipynb_file,
)
output_dir = os.path.dirname(ipynb_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(ipynb_file, 'w') as f:
nbformat.write(notebook, f)
def run(docs,
variables=None,
inputs_dir='.',
outputs_dir='.',
imports_dir='.'):
for basename, doc in docs.items():
languages = doc.get('languages', 'py java go').split()
for lang in languages:
# Read the imports defined in the docs.yaml.
imports = {
i: [os.path.join(imports_dir, path) for path in imports]
for i, imports in doc.get('imports', {}).items()
}
# Make sure the first import in section 0 is the license.md.
if 0 not in imports:
imports[0] = []
imports[0].insert(0, os.path.join(imports_dir, 'license.md'))
# Create a new notebook from the Markdown file contents.
ipynb_file = '/'.join(
[outputs_dir, '{}-{}.ipynb'.format(basename, lang)])
notebook = md2ipynb.new_notebook(
input_file=os.path.join(inputs_dir, basename + '.md'),
variables=variables,
imports=imports,
notebook_title=doc.get(
'title',
os.path.basename(basename).replace('-', ' ')),
keep_classes=['language-' + lang, 'shell-sh'],
docs_url='https://beam.apache.org/' +
basename.replace('-', ''),
docs_logo_url=docs_logo_url,
github_ipynb_url='https://github.com/apache/beam/blob/master/'
+ ipynb_file,
)
# Write the notebook to file.
output_dir = os.path.dirname(ipynb_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(ipynb_file, 'w') as f:
nbformat.write(notebook, f)
def run(docs,
root_dir,
variables=None,
inputs_dir='.',
outputs_dir='.',
imports_dir='.',
include_dir='.'):
errors = []
for basename, doc in docs.items():
languages = doc.get('languages', 'py java go').split()
for lang in languages:
# Read the imports defined in the docs.yaml.
imports = {
i: [os.path.join(imports_dir, path) for path in imports]
for i, imports in doc.get('imports', {}).items()
}
# Make sure the first import in section 0 is the license.md.
if 0 not in imports:
imports[0] = []
imports[0].insert(0, os.path.join(imports_dir, 'license.md'))
# Create a new notebook from the Markdown file contents.
input_file = basename + '.md'
ipynb_file = '/'.join(
[outputs_dir, '{}-{}.ipynb'.format(basename, lang)])
try:
notebook = md2ipynb.new_notebook(
input_file=os.path.join(inputs_dir, input_file),
variables=variables,
imports=imports,
notebook_title=doc.get(
'title',
os.path.basename(basename).replace('-', ' ')),
keep_classes=['language-' + lang, 'shell-sh'],
filter_classes='notebook-skip',
docs_url='https://beam.apache.org/' +
basename.replace('-', ''),
docs_logo_url=docs_logo_url,
github_ipynb_url=
'https://github.com/apache/beam/blob/master/' +
os.path.relpath(ipynb_file, root_dir),
include_dir=include_dir,
)
logging.info('{} succeeded'.format(input_file))
# Write the notebook to file.
output_dir = os.path.dirname(ipynb_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(ipynb_file, 'w') as f:
nbformat.write(notebook, f)
except Exception as e:
logging.error('{} failed: {}'.format(input_file, e))
errors.append((input_file, e))
if errors:
import traceback
sys.stdout.flush()
sys.stderr.flush()
print('')
print('=' * 60)
print(' Errors')
for input_file, e in errors:
print('')
print(input_file)
print('-' * len(input_file))
traceback.print_exception(type(e), e, e.__traceback__)
print('')
print('{} files processed ({} succeeded, {} failed)'.format(
len(docs),
len(docs) - len(errors), len(errors)))