def clear_notebook(path, kwargs={ "SCOPETYPE": "OPENADC", "PLATFORM": "CWLITEARM", "VERSION": "HARDWARE" }): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) # special case: if the top block isn't a parameter block, nbparameterise will: # * error out if invalid python syntax # * replace code with a parameter block (whatever we passed in with kwargs, including an empty block) # so if no parameters being changed, don't run nbparameterise if len(kwargs) > 0: orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, **kwargs) new_nb = replace_definitions(nb, params, execute=False) else: new_nb = nb co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def execute_nb2(path, **kwargs): # Reset python workspace chdir = '/'.join(os.path.abspath(path).split('/')[:-1]) os.chdir(chdir) # Execute notebook with io.open(path, mode="r", encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) # Update the parameters and run the notebook execution_date = '2018-01-01' signature = { 'dag_id': '', 'task_id': '', 'owner': '', 'nb_path': path, 'execution_date': execution_date } try: execution_date = kwargs.get('execution_date', '').strftime('%Y-%m-%d') signature['dag_id'] = str(kwargs.get('dag').dag_id) signature['task_id'] = str(kwargs.get('task').task_id) signature['owner'] = str(kwargs.get('task').owner) signature['execution_date'] = execution_date except: pass params = parameter_values(orig_parameters, execution_date=execution_date, signature=str(signature)) new_nb = replace_definitions(nb, params) # Save results back to the notebook with io.open(path, mode="w", encoding="utf-8") as f: nbformat.write(new_nb, f)
def _execute(nb, **notebook_params): orig_parameters = nbparameterise.extract_parameters(nb) # extracts parameters from FIRST cell replaced_params = nbparameterise.parameter_values(orig_parameters, **notebook_params) nb = nbparameterise.replace_definitions(nb, replaced_params) ep = nbconvert.preprocessors.ExecutePreprocessor(timeout=EXECUTION_TIMEOUT, kernel_name=KERNEL_NAME) ep.preprocess(nb) return nb
def __init__(self, path): self.path = path self.nb = nbformat.read(path, as_version=4) basename = os.path.basename(path) assert basename.endswith('.ipynb') self.nbname = basename[:-6] self.parameters = extract_parameters(self.nb) super().__init__([(r"/", MainHandler), (r"/submit", SubmissionHandler)], static_path=static_path)
def execute_notebook(nb_path, serial_number, allow_errors=True, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', **kwargs): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ notebook_dir, file_name = os.path.split(nb_path) real_path = Path(nb_path).absolute() with open(real_path) as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE=SCOPETYPE, PLATFORM=PLATFORM, **kwargs) nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3', allow_errors=allow_errors) replacements = { r'cw.scope(\(\))': 'cw.scope(sn=\'{}\')'.format(serial_number), r'chipwhisperer.scope()': 'chipwhisperer.scope(sn=\'{}\')'.format(serial_number) } rp = RegexReplacePreprocessor(replacements) ip = InLineCodePreprocessor(notebook_dir) if serial_number: # inline all code before putting in serial # numbers nb, resources = ip.preprocess(nb, {}) # add serial number to the cw.scope calls nb, resources = rp.preprocess(nb, {}) if notebook_dir: with cd(notebook_dir): nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) else: nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell for output in cell["outputs"] \ if output.output_type == "error"] export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM} return nb, errors, export_kwargs
def __init__(self, path): self.path = path with open(path) as f: self.nb = nbformat.read(f, as_version=4) basename = os.path.basename(path) assert basename.endswith('.ipynb') self.nbname = basename[:-6] self.parameters = extract_parameters(self.nb) super().__init__([ (r"/", MainHandler), (r"/submit", SubmissionHandler) ], static_path=static_path)
def notebook_to_html(notebook_path, parameters): """Run all cells and convert to static html `notebook_path`: Path to notebook `parameters`: Dict of param keys to values """ with open(os.path.join(ARGS.root, notebook_path)) as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) replaced_params = parameter_values(orig_parameters, **parameters) parametrised_nb = replace_definitions(nb, replaced_params) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(parametrised_nb, {}) html_exporter = HTMLExporter() html_exporter.template_file = 'template.tpl' (body, resources) = html_exporter.from_notebook_node(parametrised_nb) return body
def _set_params(nb, param_dict=dict(), title=None): orig_params = nbparameterise.extract_parameters(nb) params = nbparameterise.parameter_values(orig_params, **param_dict) nb = nbparameterise.replace_definitions(nb, params, execute=False) for p in params: if p.name == 'title': nb.metadata.title = p.value return nb # # if title is None: title = 'Notebook' # nb.metadata.title = title return nb
def create_new_notebook(cluster): with open("template.ipynb") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) print("Cluster", cluster) # Update the parameters and run the notebook params = parameter_values(orig_parameters, cluster_input=cluster) new_nb = replace_definitions(nb, params, execute=False) new_nb, resources = ClearOutputPreprocessor().preprocess(new_nb, resources={}) new_nb, resources = ExecutePreprocessor(timeout=1440).preprocess(new_nb, resources={}) # Save with open("/scratch/sh8tv/Project/CAD_scATAC/SVMpipeline/celltype/step3_interpretation/%s.ipynb" % str(cluster), 'w') as f: nbformat.write(new_nb, f)
def create_new_notebook(cluster): with open("template.ipynb") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) print("Cluster", cluster) # Update the parameters and run the notebook params = parameter_values(orig_parameters, cluster_input=cluster) new_nb = replace_definitions(nb, params, execute=False) new_nb, resources = ClearOutputPreprocessor().preprocess(new_nb, resources={}) new_nb, resources = ExecutePreprocessor(timeout=1440).preprocess(new_nb, resources={}) # Save with open("../Cluster%s.ipynb" % str(cluster), 'w') as f: nbformat.write(new_nb, f)
def execute_nb(path, **kwargs): # Reset python workspace chdir = '/'.join(os.path.abspath(path).split('/')[:-1]) os.chdir(chdir) # Execute notebook with io.open(path, mode="r", encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) # Update the parameters and run the notebook params = parameter_values(orig_parameters, **kwargs) new_nb = replace_definitions(nb, params) # Save results back to the notebook with io.open(path, mode="w", encoding="utf-8") as f: nbformat.write(new_nb, f)
def clear_notebook(path): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE="OPENADC", PLATFORM="CWLITEARM") new_nb = replace_definitions(nb, params, execute=False) co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def run_notebook(notebook, new_notebook, params_dict, savedir, execute_nb=True): """ Generate and run a notebook given existing notebook and dataset parameters Args: notebook (str): File name of original notebook new_notebook (str): File name of new notebook params_dict (dict): Dictionary containing name, delimiter, and composition column label for data file savedir (str): directory in which to save new notebook execute_nb (bool): Executes notebooks if True, otherwise only creates them """ with open(notebook) as f: nb_run = nbformat.read(f, as_version=4) old_params = nbparameterise.extract_parameters(nb_run) params = nbparameterise.parameter_values(old_params, **params_dict) new_nb = nbparameterise.replace_definitions(nb_run, params, execute=False) cwd = os.getcwd() with open(new_notebook, mode='wt') as f: nbformat.write(new_nb, f) #json.dump(new_nb, 'temp.ipynb') if execute_nb: ExecutePreprocessor(timeout=-1, kernel_name="python3").preprocess( new_nb, {"metadata": { "path": cwd }}) if not os.path.isdir(savedir): os.mkdir(savedir) os.chdir(savedir) with open(new_notebook, mode='wt') as f: nbformat.write(new_nb, f) os.chdir(cwd)
def create_qc_report(subject_dir, outdir): # get location of this file template_name = get_package_data_name('report_template.ipynb') template_nb = nbformat.read(template_name, as_version=4) # extract original parameters from the template notebook orig_parameters = extract_parameters(template_nb) # replace with parameters for this subject and execute notebook new_parameters = parameter_values(orig_parameters, subject_dir=str(subject_dir), outdir=str(outdir)) new_nb = replace_definitions(template_nb, new_parameters, execute=False) _ = execute(new_nb) # save notebook in subject's main output directory new_nb_name = Path(subject_dir)/outdir/"hcp_asl_report.ipynb" with open(new_nb_name, "w") as f: nbformat.write(new_nb, f)
def run_notebook_with_args(input_notebook, output_notebook, unknown): with codecs.open(input_notebook, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) params = [] args = compute_args(unknown) for k, v in args.items(): for p in orig_parameters: if p.name == k: params.append(p.with_value(v)) if len(params) > 0: nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3') out = ep.preprocess(nb, {'metadata': {'path': '.'}}) with codecs.open(output_notebook, encoding="utf-8", mode="w") as f: nbformat.write(nb, f)
def _notebook_run(path, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', **kwargs): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ html_path = Path("html/" + path + "-{}-{}".format(SCOPETYPE, PLATFORM) + ".html") real_path = Path(path) with open(real_path) as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE=SCOPETYPE, PLATFORM=PLATFORM, **kwargs) new_nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3', allow_errors=True) ep.preprocess(new_nb, {'metadata': {'path': './'}}) errors = [[i+1,output] for i,cell in enumerate(new_nb.cells) if "outputs" in cell for output in cell["outputs"]\ if output.output_type == "error"] with open(html_path, "w", encoding='utf-8') as html_file: html_exporter = HTMLExporter() body, res = html_exporter.from_notebook_node(new_nb) body = ansi2html(body) html_file.write(body) return nb, errors
def get_result(PLOT_KIND, PLOT_HIST_SIZE): path = settings.NOTEBOOKS + 'DVS_ANALYSIS.ipynb' with open(path) as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values( orig_parameters, **{ 'PLOT_KIND': PLOT_KIND, 'PLOT_HIST_SIZE': PLOT_HIST_SIZE }) new_nb = replace_definitions(nb, params, execute=False) exec(new_nb.cells[0].source) exec(new_nb.cells[2].source) exec(new_nb.cells[5].source) a = '' exec('a = ' + new_nb.cells[6].source) # a.savefig('abcd') print("--- %s seconds ---" % (time.time() - start_time)) return a
#!/usr/bin/env python3 """Example of using nbparameterise API to substitute variables in 'batch mode' """ from nbparameterise import extract_parameters, parameter_values, replace_definitions import nbformat stock_names = ['YHOO', 'MSFT', 'GOOG'] with open("Stock display.ipynb") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) for name in stock_names: print("Running for stock", name) # Update the parameters and run the notebook params = parameter_values(orig_parameters, stock=name) new_nb = replace_definitions(nb, params) # Save with open("Stock display %s.ipynb" % name, 'w') as f: nbformat.write(new_nb, f)
parser.add_argument("plot_notebook", help="the notebook the generates the plot") parser.add_argument("program", help="the program for the plot") parser.add_argument("-d", "--date", nargs=2, type=int, help="the year and month for the plot. Default: yesterday's year and month.") args = parser.parse_args() if args.date is None: # TODO: Figure out some fuzziness around the edges of the month # e.g. when the task runs in UTC, will we still not recompute on the last day yesterday = arrow.get() args.date = [yesterday.year, yesterday.month] print(f"Running at {arrow.get()} with args {args}") with open(args.plot_notebook) as f: nb = nbformat.read(f, as_version=4) # Get a list of Parameter objects orig_parameters = nbp.extract_parameters(nb) # Update one or more parameters params = nbp.parameter_values(orig_parameters, year=args.date[0], month=args.date[1], program=args.program) # Make a notebook object with these definitions new_nb = nbp.replace_definitions(nb, params, execute=False) # Execute the notebook with the new parameters nbclient.execute(new_nb)
import nbformat from nbconvert.preprocessors import ExecutePreprocessor import subprocess import sys APPNAME = 'generate_report' if ".html" in script_arguments.output: script_arguments.output = script_arguments.output[0:-5] RESULT_FILENAME = script_arguments.output + '.ipynb' print('[{}] Reading template...'.format(APPNAME)) with open("report_template.ipynb") as f: nb = nbformat.read(f, as_version=4) orig_parameters = nbparameterise.extract_parameters(nb) ep = ExecutePreprocessor(timeout=600, kernel_name=script_arguments.kernel_name) params = nbparameterise.parameter_values( orig_parameters, test_filename=script_arguments.filename, description=script_arguments.description) print('[{}] Replacing parameters...'.format(APPNAME)) new_nb = nbparameterise.replace_definitions(nb, params, execute=False) print('[{}] Executing notebook...'.format(APPNAME)) _ = ep.preprocess(new_nb, {'metadata': {'path': '.'}}) print('[{}] Saving notebook to temporary file...'.format(APPNAME))
def execute_notebook(nb_path, serial_number=None, baud=None, allow_errors=True, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', **kwargs): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ notebook_dir, file_name = os.path.split(nb_path) real_path = Path(nb_path).absolute() with open(real_path) as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE=SCOPETYPE, PLATFORM=PLATFORM, **kwargs) nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3', allow_errors=allow_errors) if serial_number or baud: ip = InLineCodePreprocessor(notebook_dir) # inline all code before doing any replacements nb, resources = ip.preprocess(nb, {}) replacements = {} if serial_number: replacements.update({ r'cw.scope(\(\))': 'cw.scope(sn=\'{}\')'.format(serial_number), r'chipwhisperer.scope()': 'chipwhisperer.scope(sn=\'{}\')'.format(serial_number) }) if baud: replacements.update({ r'program_target\(((?:[\w=\+/*\s]+\s*,\s*)*[\w=+/*]+)': r"program_target(\g<1>, baud=38400" }) # complete all regex subtitutions if replacements: rp = RegexReplacePreprocessor(replacements) nb, resources = rp.preprocess(nb, {}) if notebook_dir: with cd(notebook_dir): nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) else: nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell for output in cell["outputs"] \ if output.output_type == "error"] export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM} return nb, errors, export_kwargs
def set_parameters(nb, params_dict): orig_parameters = nbparameterise.extract_parameters(nb) params = nbparameterise.parameter_values(orig_parameters, **params_dict) new_nb = nbparameterise.replace_definitions(nb, params, execute=False) return new_nb
#!/usr/bin/env python3 """Example of using nbparameterise API to substitute variables in 'batch mode' """ # NYSE:New York Stock Exchange 纽约证券交易所 # 二十支股票: # GOOG谷歌,TSLA特斯拉,AAPL苹果,AMZN亚马逊,MSFT微软,NVDA英伟达,INTC英特尔,QCOM高通,WDC西部数据,AMD超威,ORCL甲骨文,SAP思爱普,CSCO思科,BABA阿里巴巴,NOK诺基亚,T美国电话电报公司,VZ微讯,FB脸书,TWTR推特,SNAP色拉布 from nbparameterise import extract_parameters, parameter_values, replace_definitions import nbformat stock_names = ['GOOG','TSLA','AAPL','AMZN','MSFT','NVDA','INTC','QCOM','WDC','AMD','ORCL','SAP','CSCO','BABA', 'NOK','T','VZ','FB','TWTR','SNAP'] with open("StockRecommendationSystem.ipynb") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) for name in stock_names: print("Running for stock", name) # Update the parameters and run the notebook params = parameter_values(orig_parameters, stock=name) new_nb = replace_definitions(nb, params) # Save with open("StockRecommendationSystem %s.ipynb" % name, 'w') as f: nbformat.write(new_nb, f)
def execute_notebook(nb_path, serial_number=None, baud=None, hw_location=None, allow_errors=True, SCOPETYPE='OPENADC', PLATFORM='CWLITEARM', logger=None, **kwargs): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ notebook_dir, file_name = os.path.split(nb_path) real_path = Path(nb_path).absolute() if logger is None: logger = test_logger with open(real_path, encoding='utf-8') as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE=SCOPETYPE, PLATFORM=PLATFORM, **kwargs) kwargs['SCOPETYPE'] = SCOPETYPE kwargs['PLATFORM'] = PLATFORM put_all_kwargs_in_notebook(params, logger=logger, **kwargs) nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3', allow_errors=allow_errors) if serial_number or baud or hw_location: ip = InLineCodePreprocessor(notebook_dir) # inline all code before doing any replacements nb, resources = ip.preprocess(nb, {}) replacements = {} if hw_location and serial_number: print("Make error for this later") if serial_number: replacements.update({ r'cw.scope(\(\))': 'cw.scope(sn=\'{}\')'.format(serial_number), r'chipwhisperer.scope()': 'chipwhisperer.scope(sn=\'{}\')'.format(serial_number) }) if baud: # replacements.update({ # r'program_target\(((?:[\w=\+/*\s]+\s*,\s*)*[\w=+/*]+)': r"program_target(\g<1>, baud=38400" # }) replacements.update( {r'(program_target\(.*)\)': r'\g<1>, baud={})'.format(baud)}) if hw_location: replacements.update({ r'cw.scope(\(\))': 'cw.scope(hw_location={})'.format(hw_location), r'chipwhisperer.scope()': 'chipwhisperer.scope(hw_location={})'.format(hw_location) }) # %matplotlib notebook won't show up in blank plots # so replace with %matplotlib inline for now replacements.update({'%matplotlib notebook': '%matplotlib inline'}) # complete all regex subtitutions if replacements: rp = RegexReplacePreprocessor(replacements) nb, resources = rp.preprocess(nb, {}) if notebook_dir: with cd(notebook_dir): nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) else: nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) errors = [[i + 1, output] for i, cell in enumerate(nb.cells) if "outputs" in cell for output in cell["outputs"] \ if output.output_type == "error"] export_kwargs = {'SCOPETYPE': SCOPETYPE, 'PLATFORM': PLATFORM} return nb, errors, export_kwargs