def execute_nb2(path, **kwargs):
# Reset python workspace
chdir = '/'.join(os.path.abspath(path).split('/')[:-1])
os.chdir(chdir)
# Execute notebook
with io.open(path, mode="r", encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
# Update the parameters and run the notebook
execution_date = '2018-01-01'
signature = {
'dag_id': '',
'task_id': '',
'owner': '',
'nb_path': path,
'execution_date': execution_date
}
try:
execution_date = kwargs.get('execution_date', '').strftime('%Y-%m-%d')
signature['dag_id'] = str(kwargs.get('dag').dag_id)
signature['task_id'] = str(kwargs.get('task').task_id)
signature['owner'] = str(kwargs.get('task').owner)
signature['execution_date'] = execution_date
except:
pass
params = parameter_values(orig_parameters,
execution_date=execution_date,
signature=str(signature))
new_nb = replace_definitions(nb, params)
# Save results back to the notebook
with io.open(path, mode="w", encoding="utf-8") as f:
nbformat.write(new_nb, f)
def clear_notebook(path,
kwargs={
"SCOPETYPE": "OPENADC",
"PLATFORM": "CWLITEARM",
"VERSION": "HARDWARE"
}):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
# special case: if the top block isn't a parameter block, nbparameterise will:
# * error out if invalid python syntax
# * replace code with a parameter block (whatever we passed in with kwargs, including an empty block)
# so if no parameters being changed, don't run nbparameterise
if len(kwargs) > 0:
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters, **kwargs)
new_nb = replace_definitions(nb, params, execute=False)
else:
new_nb = nb
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def _execute(nb, **notebook_params):
orig_parameters = nbparameterise.extract_parameters(nb) # extracts parameters from FIRST cell
replaced_params = nbparameterise.parameter_values(orig_parameters, **notebook_params)
nb = nbparameterise.replace_definitions(nb, replaced_params)
ep = nbconvert.preprocessors.ExecutePreprocessor(timeout=EXECUTION_TIMEOUT, kernel_name=KERNEL_NAME)
ep.preprocess(nb)
return nb
def execute_notebook(nb_path,
serial_number,
allow_errors=True,
SCOPETYPE='OPENADC',
PLATFORM='CWLITEARM',
**kwargs):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
notebook_dir, file_name = os.path.split(nb_path)
real_path = Path(nb_path).absolute()
with open(real_path) as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE=SCOPETYPE,
PLATFORM=PLATFORM,
**kwargs)
nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None,
kernel_name='python3',
allow_errors=allow_errors)
replacements = {
r'cw.scope(\(\))':
'cw.scope(sn=\'{}\')'.format(serial_number),
r'chipwhisperer.scope()':
'chipwhisperer.scope(sn=\'{}\')'.format(serial_number)
}
rp = RegexReplacePreprocessor(replacements)
ip = InLineCodePreprocessor(notebook_dir)
if serial_number:
# inline all code before putting in serial
# numbers
nb, resources = ip.preprocess(nb, {})
# add serial number to the cw.scope calls
nb, resources = rp.preprocess(nb, {})
if notebook_dir:
with cd(notebook_dir):
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
else:
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell
for output in cell["outputs"] \
if output.output_type == "error"]
export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM}
return nb, errors, export_kwargs
def post(self):
defined = []
for v in self.application.parameters:
if v.type is bool:
inp = v.with_value(self.get_argument(v.name, default='off') == 'on')
else:
inp = v.with_value(v.type(self.get_argument(v.name)))
defined.append(inp)
res = {'path': os.path.dirname(self.application.path)}
nb = replace_definitions(self.application.nb, defined, execute_resources=res)
output, _ = HTMLExporter().from_notebook_node(nb, res)
self.write(output)
def post(self):
defined = []
for v in self.application.parameters:
if v.type is bool:
inp = v.with_value(
self.get_argument(v.name, default='off') == 'on')
else:
inp = v.with_value(v.type(self.get_argument(v.name)))
defined.append(inp)
nb = replace_definitions(self.application.nb, defined)
nb = execute(nb, cwd=os.path.dirname(self.application.path))
output, _ = HTMLExporter().from_notebook_node(nb)
self.write(output)
def notebook_to_html(notebook_path, parameters):
"""Run all cells and convert to static html
`notebook_path`: Path to notebook
`parameters`: Dict of param keys to values
"""
with open(os.path.join(ARGS.root, notebook_path)) as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
replaced_params = parameter_values(orig_parameters, **parameters)
parametrised_nb = replace_definitions(nb, replaced_params)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(parametrised_nb, {})
html_exporter = HTMLExporter()
html_exporter.template_file = 'template.tpl'
(body, resources) = html_exporter.from_notebook_node(parametrised_nb)
return body
def _set_params(nb, param_dict=dict(), title=None):
orig_params = nbparameterise.extract_parameters(nb)
params = nbparameterise.parameter_values(orig_params, **param_dict)
nb = nbparameterise.replace_definitions(nb, params, execute=False)
for p in params:
if p.name == 'title':
nb.metadata.title = p.value
return nb
#
#
if title is None:
title = 'Notebook'
#
nb.metadata.title = title
return nb
def create_new_notebook(cluster):
with open("template.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
print("Cluster", cluster)
# Update the parameters and run the notebook
params = parameter_values(orig_parameters, cluster_input=cluster)
new_nb = replace_definitions(nb, params, execute=False)
new_nb, resources = ClearOutputPreprocessor().preprocess(new_nb, resources={})
new_nb, resources = ExecutePreprocessor(timeout=1440).preprocess(new_nb, resources={})
# Save
with open("/scratch/sh8tv/Project/CAD_scATAC/SVMpipeline/celltype/step3_interpretation/%s.ipynb" % str(cluster), 'w') as f:
nbformat.write(new_nb, f)
def create_new_notebook(cluster):
with open("template.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
print("Cluster", cluster)
# Update the parameters and run the notebook
params = parameter_values(orig_parameters, cluster_input=cluster)
new_nb = replace_definitions(nb, params, execute=False)
new_nb, resources = ClearOutputPreprocessor().preprocess(new_nb, resources={})
new_nb, resources = ExecutePreprocessor(timeout=1440).preprocess(new_nb, resources={})
# Save
with open("../Cluster%s.ipynb" % str(cluster), 'w') as f:
nbformat.write(new_nb, f)
def execute_nb(path, **kwargs):
# Reset python workspace
chdir = '/'.join(os.path.abspath(path).split('/')[:-1])
os.chdir(chdir)
# Execute notebook
with io.open(path, mode="r", encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
# Update the parameters and run the notebook
params = parameter_values(orig_parameters, **kwargs)
new_nb = replace_definitions(nb, params)
# Save results back to the notebook
with io.open(path, mode="w", encoding="utf-8") as f:
nbformat.write(new_nb, f)
def clear_notebook(path):
real_path = Path(path)
body = ""
with open(real_path, "r", encoding="utf-8") as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE="OPENADC",
PLATFORM="CWLITEARM")
new_nb = replace_definitions(nb, params, execute=False)
co = ClearOutputPreprocessor()
exporter = NotebookExporter()
node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}})
body, resources = exporter.from_notebook_node(node, resources)
with open(real_path, "w", encoding="utf-8") as nbfile:
nbfile.write(body)
def run_notebook(notebook,
new_notebook,
params_dict,
savedir,
execute_nb=True):
"""
Generate and run a notebook given existing notebook and dataset parameters
Args:
notebook (str): File name of original notebook
new_notebook (str): File name of new notebook
params_dict (dict): Dictionary containing name, delimiter, and composition column label for data file
savedir (str): directory in which to save new notebook
execute_nb (bool): Executes notebooks if True, otherwise only creates them
"""
with open(notebook) as f:
nb_run = nbformat.read(f, as_version=4)
old_params = nbparameterise.extract_parameters(nb_run)
params = nbparameterise.parameter_values(old_params, **params_dict)
new_nb = nbparameterise.replace_definitions(nb_run, params, execute=False)
cwd = os.getcwd()
with open(new_notebook, mode='wt') as f:
nbformat.write(new_nb, f)
#json.dump(new_nb, 'temp.ipynb')
if execute_nb:
ExecutePreprocessor(timeout=-1, kernel_name="python3").preprocess(
new_nb, {"metadata": {
"path": cwd
}})
if not os.path.isdir(savedir):
os.mkdir(savedir)
os.chdir(savedir)
with open(new_notebook, mode='wt') as f:
nbformat.write(new_nb, f)
os.chdir(cwd)
def create_qc_report(subject_dir, outdir):
# get location of this file
template_name = get_package_data_name('report_template.ipynb')
template_nb = nbformat.read(template_name, as_version=4)
# extract original parameters from the template notebook
orig_parameters = extract_parameters(template_nb)
# replace with parameters for this subject and execute notebook
new_parameters = parameter_values(orig_parameters,
subject_dir=str(subject_dir),
outdir=str(outdir))
new_nb = replace_definitions(template_nb, new_parameters, execute=False)
_ = execute(new_nb)
# save notebook in subject's main output directory
new_nb_name = Path(subject_dir)/outdir/"hcp_asl_report.ipynb"
with open(new_nb_name, "w") as f:
nbformat.write(new_nb, f)
def run_notebook_with_args(input_notebook, output_notebook, unknown):
with codecs.open(input_notebook, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
params = []
args = compute_args(unknown)
for k, v in args.items():
for p in orig_parameters:
if p.name == k:
params.append(p.with_value(v))
if len(params) > 0:
nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None, kernel_name='python3')
out = ep.preprocess(nb, {'metadata': {'path': '.'}})
with codecs.open(output_notebook, encoding="utf-8", mode="w") as f:
nbformat.write(nb, f)
def _notebook_run(path, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', **kwargs):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
html_path = Path("html/" + path + "-{}-{}".format(SCOPETYPE, PLATFORM) +
".html")
real_path = Path(path)
with open(real_path) as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE=SCOPETYPE,
PLATFORM=PLATFORM,
**kwargs)
new_nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None,
kernel_name='python3',
allow_errors=True)
ep.preprocess(new_nb, {'metadata': {'path': './'}})
errors = [[i+1,output] for i,cell in enumerate(new_nb.cells) if "outputs" in cell
for output in cell["outputs"]\
if output.output_type == "error"]
with open(html_path, "w", encoding='utf-8') as html_file:
html_exporter = HTMLExporter()
body, res = html_exporter.from_notebook_node(new_nb)
body = ansi2html(body)
html_file.write(body)
return nb, errors
def get_result(PLOT_KIND, PLOT_HIST_SIZE):
path = settings.NOTEBOOKS + 'DVS_ANALYSIS.ipynb'
with open(path) as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(
orig_parameters, **{
'PLOT_KIND': PLOT_KIND,
'PLOT_HIST_SIZE': PLOT_HIST_SIZE
})
new_nb = replace_definitions(nb, params, execute=False)
exec(new_nb.cells[0].source)
exec(new_nb.cells[2].source)
exec(new_nb.cells[5].source)
a = ''
exec('a = ' + new_nb.cells[6].source)
# a.savefig('abcd')
print("--- %s seconds ---" % (time.time() - start_time))
return a
#!/usr/bin/env python3
"""Example of using nbparameterise API to substitute variables in 'batch mode'
"""
from nbparameterise import extract_parameters, parameter_values, replace_definitions
import nbformat
stock_names = ['YHOO', 'MSFT', 'GOOG']
with open("Stock display.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
for name in stock_names:
print("Running for stock", name)
# Update the parameters and run the notebook
params = parameter_values(orig_parameters, stock=name)
new_nb = replace_definitions(nb, params)
# Save
with open("Stock display %s.ipynb" % name, 'w') as f:
nbformat.write(new_nb, f)
print('[{}] Reading template...'.format(APPNAME))
with open("report_template.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = nbparameterise.extract_parameters(nb)
ep = ExecutePreprocessor(timeout=600, kernel_name=script_arguments.kernel_name)
params = nbparameterise.parameter_values(
orig_parameters,
test_filename=script_arguments.filename,
description=script_arguments.description)
print('[{}] Replacing parameters...'.format(APPNAME))
new_nb = nbparameterise.replace_definitions(nb, params, execute=False)
print('[{}] Executing notebook...'.format(APPNAME))
_ = ep.preprocess(new_nb, {'metadata': {'path': '.'}})
print('[{}] Saving notebook to temporary file...'.format(APPNAME))
with open(RESULT_FILENAME, 'wt') as f:
nbformat.write(new_nb, f)
try:
print('[{}] Converting to html...'.format(APPNAME))
subprocess.call(['jupyter', 'nbconvert', RESULT_FILENAME, '--to', 'html'])
finally:
print('[{}] Deleting {}...'.format(APPNAME, RESULT_FILENAME))
if sys.platform == 'win32':
def execute_notebook(nb_path,
serial_number=None,
baud=None,
allow_errors=True,
SCOPETYPE='OPENADC',
PLATFORM='CWLITEARM',
**kwargs):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
notebook_dir, file_name = os.path.split(nb_path)
real_path = Path(nb_path).absolute()
with open(real_path) as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE=SCOPETYPE,
PLATFORM=PLATFORM,
**kwargs)
nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None,
kernel_name='python3',
allow_errors=allow_errors)
if serial_number or baud:
ip = InLineCodePreprocessor(notebook_dir)
# inline all code before doing any replacements
nb, resources = ip.preprocess(nb, {})
replacements = {}
if serial_number:
replacements.update({
r'cw.scope(\(\))':
'cw.scope(sn=\'{}\')'.format(serial_number),
r'chipwhisperer.scope()':
'chipwhisperer.scope(sn=\'{}\')'.format(serial_number)
})
if baud:
replacements.update({
r'program_target\(((?:[\w=\+/*\s]+\s*,\s*)*[\w=+/*]+)':
r"program_target(\g<1>, baud=38400"
})
# complete all regex subtitutions
if replacements:
rp = RegexReplacePreprocessor(replacements)
nb, resources = rp.preprocess(nb, {})
if notebook_dir:
with cd(notebook_dir):
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
else:
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell
for output in cell["outputs"] \
if output.output_type == "error"]
export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM}
return nb, errors, export_kwargs
nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
orig_parameters = extract_parameters(nb)
# Parameters to be passed
param_dictionary = {
'PLOT_HIST_SIZE': 30,
'PLOT_KIND': 'bar',
'SCATTER_PLOT_LIMIT': 50,
'BUBBLE_SIZE': 300.0,
'RANK': 30
}
params = parameter_values(orig_parameters, **param_dictionary)
new_nb = replace_definitions(nb, params, execute=True)
print(new_nb.cells[4].outputs)
# ep = ExecutePreprocessor(timeout=2000, kernel_name='python3')
# ep.preprocess(new_nb, resources={'metadata': {'path': ''}})
with open(TARGET_IPYNB_FILE, 'wt') as f:
nbformat.write(new_nb, f)
# with open(TARGET_IPYNB_FILE) as f:
# nb = nbformat.read(f, as_version=4)
# print(nb.cells[3].outputs[0].data['text/plain'])
# figure = nb.cells[3].outputs[0].data['text/plain']
# print(type(figure))
print("\n--- %s seconds ---" % (time.time() - start_time))
def set_parameters(nb, params_dict):
orig_parameters = nbparameterise.extract_parameters(nb)
params = nbparameterise.parameter_values(orig_parameters, **params_dict)
new_nb = nbparameterise.replace_definitions(nb, params, execute=False)
return new_nb
# Extract parameters in source notebook
with open("source.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
# Parameters to replace
model_names = [
'model3-small-search-space4', 'model3-small-search-space5'
]
# Create new notebooks with replaced parameters
for name in model_names:
print("Running for model:", name)
# Update the parameters
params = parameter_values(orig_parameters, model_name=name)
new_nb = replace_definitions(nb, params, execute=False)
# Save
new_nb_fp = f"new_notebook_with_param_{name}.ipynb"
with open(new_nb_fp, 'w') as f:
nbformat.write(new_nb, f)
# Execute new notebook
os.system(f"jupyter nbconvert --execute --ExecutePreprocessor.timeout=-1 --to notebook --inplace {new_nb_fp}")
# Convert to html
os.system(f"jupyter nbconvert --output-dir='./html' --to html {new_nb_fp}")
````
```
#!/usr/bin/env python3
"""Example of using nbparameterise API to substitute variables in 'batch mode'
"""
# NYSE：New York Stock Exchange 纽约证券交易所
# 二十支股票：
# GOOG谷歌,TSLA特斯拉,AAPL苹果,AMZN亚马逊,MSFT微软,NVDA英伟达,INTC英特尔,QCOM高通,WDC西部数据,AMD超威,ORCL甲骨文,SAP思爱普,CSCO思科,BABA阿里巴巴,NOK诺基亚,T美国电话电报公司,VZ微讯,FB脸书,TWTR推特,SNAP色拉布
from nbparameterise import extract_parameters, parameter_values, replace_definitions
import nbformat
stock_names = ['GOOG','TSLA','AAPL','AMZN','MSFT','NVDA','INTC','QCOM','WDC','AMD','ORCL','SAP','CSCO','BABA',
'NOK','T','VZ','FB','TWTR','SNAP']
with open("StockRecommendationSystem.ipynb") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
for name in stock_names:
print("Running for stock", name)
# Update the parameters and run the notebook
params = parameter_values(orig_parameters, stock=name)
new_nb = replace_definitions(nb, params)
# Save
with open("StockRecommendationSystem %s.ipynb" % name, 'w') as f:
nbformat.write(new_nb, f)
def execute_notebook(nb_path,
serial_number=None,
baud=None,
hw_location=None,
allow_errors=True,
SCOPETYPE='OPENADC',
PLATFORM='CWLITEARM',
logger=None,
**kwargs):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
notebook_dir, file_name = os.path.split(nb_path)
real_path = Path(nb_path).absolute()
if logger is None:
logger = test_logger
with open(real_path, encoding='utf-8') as nbfile:
nb = nbformat.read(nbfile, as_version=4)
orig_parameters = extract_parameters(nb)
params = parameter_values(orig_parameters,
SCOPETYPE=SCOPETYPE,
PLATFORM=PLATFORM,
**kwargs)
kwargs['SCOPETYPE'] = SCOPETYPE
kwargs['PLATFORM'] = PLATFORM
put_all_kwargs_in_notebook(params, logger=logger, **kwargs)
nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None,
kernel_name='python3',
allow_errors=allow_errors)
if serial_number or baud or hw_location:
ip = InLineCodePreprocessor(notebook_dir)
# inline all code before doing any replacements
nb, resources = ip.preprocess(nb, {})
replacements = {}
if hw_location and serial_number:
print("Make error for this later")
if serial_number:
replacements.update({
r'cw.scope(\(\))':
'cw.scope(sn=\'{}\')'.format(serial_number),
r'chipwhisperer.scope()':
'chipwhisperer.scope(sn=\'{}\')'.format(serial_number)
})
if baud:
# replacements.update({
# r'program_target\(((?:[\w=\+/*\s]+\s*,\s*)*[\w=+/*]+)': r"program_target(\g<1>, baud=38400"
# })
replacements.update(
{r'(program_target\(.*)\)': r'\g<1>, baud={})'.format(baud)})
if hw_location:
replacements.update({
r'cw.scope(\(\))':
'cw.scope(hw_location={})'.format(hw_location),
r'chipwhisperer.scope()':
'chipwhisperer.scope(hw_location={})'.format(hw_location)
})
# %matplotlib notebook won't show up in blank plots
# so replace with %matplotlib inline for now
replacements.update({'%matplotlib notebook': '%matplotlib inline'})
# complete all regex subtitutions
if replacements:
rp = RegexReplacePreprocessor(replacements)
nb, resources = rp.preprocess(nb, {})
if notebook_dir:
with cd(notebook_dir):
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
else:
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell
for output in cell["outputs"] \
if output.output_type == "error"]
export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM}
return nb, errors, export_kwargs