def md2ipynb(): assert len(sys.argv) == 3, 'usage: input.md output.rst' (src_fn, input_fn, output_fn) = sys.argv # timeout for each notebook, in sec timeout = 20 * 60 # if enable evaluation do_eval = int(os.environ.get('EVAL', True)) reader = notedown.MarkdownReader() with open(input_fn, 'r') as f: notebook = reader.read(f) notebook['metadata'].update({'language_info': { 'name': 'R' }}) # need to add language info for syntax highlight if do_eval: tic = time.time() executor = ExecutePreprocessor(timeout=timeout, kernel_name='ir') print('%s: Evaluated %s in %f sec' % (src_fn, input_fn, time.time() - tic)) try: notebook, resources = executor.preprocess(notebook, resources={}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % input_fn msg += 'See notebook "%s" for the traceback.' % output_fn print(msg) raise finally: with open(output_fn, 'w') as f: f.write(nbformat.writes(notebook).encode('utf8')) print('%s: Write results into %s' % (src_fn, output_fn))
def run_notebook(self, name): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now().strftime('%s') # execute try: ep = ExecutePreprocessor() ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = str(e) else: status = 'OK' # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', {}) job_runs[ts] = status meta_job.attributes['job_runs'] = job_runs meta_job.save() return meta_job
def test_demo(self): import nbformat from nbconvert.preprocessors.execute import ExecutePreprocessor here = os.path.dirname(__file__) demo = os.path.join(here,'Demo.ipynb') nb = nbformat.read(demo, as_version=4) pp = ExecutePreprocessor() pp.allow_errors = False pp.preprocess(nb, resources={})
def run_notebook(self, name, event=None): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute ep_kwargs = {'timeout': None} ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: if not meta_job.kind_meta.get('keep_output', False): resources = { } # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results
def pYPKa_ZE_ipynb_generator(tp, dir_="pYPKa_ZE_vectors"): cwd = os.getcwd() try: os.makedirs(dir_) except OSError as exception: if exception.errno == errno.EEXIST: pass else: print("The {} directory could not be created".format(dir_)) return None os.chdir(dir_) with open("standard_primers.txt","w") as f: f.write(read_data_file("standard_primers.txt")) with open("pYPKa.gb","w") as f: f.write(read_data_file("pYPKa.gb")) with open("pYPK_ZE.png","w") as f: f.write(read_bin_file("pYPK_ZE.png")) with open(tp.id+".gb","w") as f: f.write(tp.format("gb")) nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") name = "pYPKa_ZE_{}.ipynb".format(tp.id) obj = notedown.MarkdownReader() nb = obj.to_notebook(nbtemp.format(tp=tp.id)) pp = ExecutePreprocessor() pp.timeout = 120 # seconds pp.interrupt_on_timeout = True shell = InteractiveShell.instance() nb_executed, resources = pp.preprocess(nb, resources={}) g={} l={} from io import StringIO old_stdout = sys.stdout redirected_output = sys.stdout = StringIO() for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) sys.stdout = old_stdout nbformat.write(nb, name) os.chdir(cwd) return FileLinks(dir_)
def executenb(nb, cwd=None, km=None, **kwargs): """Execute a notebook and embed widget state.""" resources = {} if cwd is not None: resources['metadata'] = {'path': cwd} ep = ExecutePreprocessor(**kwargs) with ep.setup_preprocessor(nb, resources, km=km): ep.log.info("Executing notebook with kernel: %s" % ep.kernel_name) nb, resources = super(ExecutePreprocessor, ep).preprocess(nb, resources) nb.metadata.language_info = language_info(ep) widgets = extract_widget_state(ep) if widgets: nb.metadata.widgets = {WIDGET_STATE_MIMETYPE: widgets}
def run_notebook(self, name, event=None): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute try: ep = ExecutePreprocessor() ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' # record results meta_results = self.put( notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts return meta_job.save()
def execute_notebook(notebook): # Configure the notebook executor and then run the notebook # given, returning the results to the call c = Config() c.ScriptExporter.preprocessors = [NarrativePreprocessor] nar_templates = os.path.join(os.environ.get('NARRATIVE_DIR', '.'), 'src', 'biokbase', 'narrative', 'exporter', 'templates') c.TemplateExporter.template_path = ['.', nar_templates] # Initialize the notebook execution object, and run the notebook. If a # timeout (in seconds) is defined in KB_CELL_TIMEOUT, use that for # how long we allow a cell to run before timing it out, otherwise use # a default value of 60 minutes. # /tmp is the directory where the notebook will be run. if 'KB_CELL_TIMEOUT' in os.environ: timeout = int(os.environ['KB_CELL_TIMEOUT']) else: timeout = 3600 ep = ExecutePreprocessor(timeout=timeout) resources = {'metadata': {'path': '/tmp'}} return (ep.preprocess(notebook, resources))
def JKConnect(self) -> None: runtime_dir = pathlib.Path(jupyter_core.paths.jupyter_runtime_dir()) connection_files = runtime_dir.glob("kernel-*.json") source = '\n'.join( connection_file.name.lstrip('kernel-').rstrip('.json') + ' ' + datetime.fromtimestamp(connection_file.stat().st_ctime).strftime( "%m/%d %H:%M") for connection_file in connection_files) proc = subprocess.run("fzf-tmux|awk '{print $1}'", input=source, stdout=PIPE, shell=True, text=True) connection_file = 'kernel-%s.json' % proc.stdout.strip() connection_file = runtime_dir.joinpath(connection_file).as_posix() kc = BlockingKernelClient() try: kc.load_connection_file(connection_file) kc.execute_interactive('', timeout=1) except (TimeoutError, FileNotFoundError): self.nvim.command("echoerr 'Selected connection is dead!'") else: self.executor = ExecutePreprocessor() self.executor.kc = kc self.nvim.command("echo 'Successfully connected!'")
def test_all_notebooks(self): """ Test all notebooks except blacklist. Blacklisted notebooks take too long.""" print("testing all notebooks") blacklist = ["SyntheticData.ipynb"] pythonkernel = "python" + str(sys.version_info[0]) this_dir = os.path.dirname(__file__) nbpath = os.path.join(this_dir, "../notebooks/") # see http://nbconvert.readthedocs.io/en/stable/execute_api.html ep = ExecutePreprocessor( timeout=120, kernel_name=pythonkernel, interrupt_on_timeout=True ) lfiles = glob.glob(nbpath + "*.ipynb") for notebook_filename in lfiles: if os.path.basename(notebook_filename) not in blacklist: print(">> Testing notebook", notebook_filename) t = time.time() self._execNotebook(ep, notebook_filename, nbpath) print(notebook_filename, "took %g seconds." % (time.time() - t))
class Executor(object): def __init__(self, nvim: Nvim) -> None: self.nvim = nvim # self.html_path = tempfile.mktemp(suffix=".html") self.html_path = '/tmp/executor.html' self.project_root = pathlib.Path(__file__).parent / ('../' * 3) self.executor = None self.html_exporter = nbconvert.exporters.HTMLExporter() def print(self, txt: Any) -> None: txt = str(txt) self.nvim.command('new') for line in txt.splitlines(): self.nvim.current.buffer.append(line) @staticmethod def parse_traceback(line: str) -> Tuple[str, int, str]: f = line.lstrip('\033[0;32m').split('\033')[0] l = int(line.split('-> ')[-1].split('\033')[0]) m = line.split('in \033[0;36m')[-1].split('\033')[0] return f, l, m @staticmethod def code_surjection(buffer: Buffer, nb: NotebookNode) -> List[Tuple[int, int]]: surjection = [(-1, -1)] * len(buffer) i = 0 for c, cell in enumerate(nb.cells): for l, line in enumerate(cell.source.splitlines()): while line != buffer[i]: i += 1 surjection[i] = (c, l) return surjection @pynvim.command('JNConnect') def JKConnect(self) -> None: runtime_dir = pathlib.Path(jupyter_core.paths.jupyter_runtime_dir()) connection_files = runtime_dir.glob("kernel-*.json") source = '\n'.join( connection_file.name.lstrip('kernel-').rstrip('.json') + ' ' + datetime.fromtimestamp(connection_file.stat().st_ctime).strftime( "%m/%d %H:%M") for connection_file in connection_files) proc = subprocess.run("fzf-tmux|awk '{print $1}'", input=source, stdout=PIPE, shell=True, text=True) connection_file = 'kernel-%s.json' % proc.stdout.strip() connection_file = runtime_dir.joinpath(connection_file).as_posix() kc = BlockingKernelClient() try: kc.load_connection_file(connection_file) kc.execute_interactive('', timeout=1) except (TimeoutError, FileNotFoundError): self.nvim.command("echoerr 'Selected connection is dead!'") else: self.executor = ExecutePreprocessor() self.executor.kc = kc self.nvim.command("echo 'Successfully connected!'") @pynvim.command('JNRun', sync=False) def JNRun(self) -> None: current_line = self.nvim.current.line buffer = '\n'.join(self.nvim.current.buffer) nb = jupytext.reads(buffer, {'extension': '.py'}) surjection = self.code_surjection(self.nvim.current.buffer, nb) current_line = self.nvim.call('line', '.') - 1 current_cell = surjection[current_line][0] if current_cell == -1: self.nvim.command('echoerr "Current line is out of cell."') return # cell execution _, nb.cells[current_cell].outputs = self.executor.run_cell( nb.cells[current_cell], current_cell) #nb.cells[current_cell], resources = self.executor.preprocess_cell(nb.cells[current_cell], {}, current_cell) # error handling for output in nb.cells[current_cell].outputs: if output.output_type == 'error': f, l, m = self.parse_traceback(output.traceback[2]) f = self.nvim.current.buffer.name l = surjection.index((current_cell, l - 1)) + 1 self.nvim.command('enew|setl nohidden|setl bt=nofile') self.nvim.current.buffer.append(' File "%s", line %d, in %s' % (f, l, m)) for line in output.traceback[3:-1]: f, l, m = self.parse_traceback(line) self.nvim.current.buffer.append( ' File "%s", line %d, in %s' % (f, l, m)) self.nvim.command('compiler python|cbuffer|cw|set hidden') break # html export script, resources = self.html_exporter.from_notebook_node(nb) script = script.splitlines() script.insert( 3, '<script type="text/javascript" src="http://livejs.com/live.js"></script>' ) script = '\n'.join(script) with open(self.html_path, "w") as f: f.write(script) @pynvim.command('JNDevTest', nargs='*', sync=True) def JNDevTest(self, args): self.nvim.command('UpdateRemotePlugins') subprocess.call('tmux split -h "nvim %s"' % (self.project_root / 'test/test.py'), shell=True)
gbref=gbref, gblink=gblink, templatesize=templatesize, insertseguid=insertseguid, finalcseguidZ=finalcseguidZ, finalcseguidE=finalcseguidE, fpn=fp.name, fps=fp.seq, rpn=rp.name, rps=rp.seq) obj = notedown.MarkdownReader() nb = obj.to_notebook(content) pp = ExecutePreprocessor(timeout=600, kernel_name='python3') pp.timeout = 120 # seconds pp.interrupt_on_timeout = True pp.preprocess(nb, resources={}) with open(newname, 'wt') as f: nbformat.write(nb, f) #os.chdir(cwd) # with open("README_template.md", "r", encoding="utf8") as f: # t=f.read() # table = "| No. | TP | Promoter vector | Terminator vector | Jupyter nb |\n" # table+= "|-----|----|-----------------|-------------------|------------|\n"
def run(notebook, timeout=30): executor = ExecutePreprocessor(timeout=timeout) notebook, resources = executor.preprocess(notebook, resources={})
def run(notebook, timeout=30, store_widget_state=True): executor = ExecutePreprocessor(timeout=timeout, store_widget_state=store_widget_state) notebook, resources = executor.preprocess(notebook, resources={})
def run_code(source: tuple, kernel_name='python3'): """Run code blocks inside a jupyter kernel. Args: source (tuple): source code blocks kernel_name: name of the kernel (form the kernel spec) to be created """ import IPython if pkg_version.parse(IPython.__version__) < pkg_version.parse('7.6.0'): raise RuntimeError("IPython version {} not supported." " Kale requires at least version 7.6.0.".format( IPython.__version__)) # new notebook spec = get_kernel_spec(kernel_name) notebook = nbformat.v4.new_notebook( metadata={ 'kernelspec': { 'display_name': spec.display_name, 'language': spec.language, 'name': kernel_name, } }) notebook.cells = [nbformat.v4.new_code_cell(s) for s in source] # these parameters are passed to nbconvert.ExecutePreprocessor jupyter_execute_kwargs = dict(timeout=-1, allow_errors=True, store_widget_state=True) resources = {} # cwd: If supplied, the kernel will run in this directory # resources['metadata'] = {'path': cwd} ep = ExecutePreprocessor(**jupyter_execute_kwargs) km = ep.kernel_manager_class(kernel_name=kernel_name, config=ep.config) # start_kernel supports several additional arguments via **kw km.start_kernel(extra_arguments=ep.extra_arguments) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=60) except RuntimeError: kc.stop_channels() raise kc.allow_stdin = False def signal_handler(_signal, _frame): raise KaleKernelException() # this signal is used by the thread in case an error message is received # by the kernel. Running sys.exit() inside the thread would terminate # just the thread itself, not the main process. Calling os._exit() can be # dangerous as the process is killed instantly (files and connections are # not closed, for example). With a signal we can capture the ExitCommand # exception from the main process and exit gracefully. signal.signal(signal.SIGUSR1, signal_handler) # start separate thread in to capture and print stdout, stderr, errors. # daemon mode will make the watcher thread die when the main one returns. x = threading.Thread(target=capture_streams, args=( kc, True, ), daemon=True) x.start() try: # start preprocessor: run each code cell and capture the output ep.preprocess(notebook, resources, km=km) except KaleKernelException: # exit gracefully with error sys.exit(-1) # Give some time to the stream watcher thread to receive all messages from # the kernel before shutting down. time.sleep(1) km.shutdown_kernel() result = process_outputs(notebook.cells) return result
def pathway(pth, dir_="ypkassembly", pYPKa_A=True, print=print): if len(pth)==0: # pth has to contain some sequences print("No of sequences found.") return None, None names = [s.name for s in pth] # sequence names has to be unique #if len(names)>len(set(names)): # print("Gene names are not unique. Please rename sequences so that each sequence has a unique name.\n") # print("Gene names parsed from Data page:\n\n") # for name in names: # print(name) # return None, None log="" pYPK0 = read(read_data_file("pYPK0.gb")) pYPKa = read(read_data_file("pYPKa.gb")) from Bio.Restriction import ZraI, AjiI, EcoRV files = {"standard_primers.txt" : read_data_file("standard_primers.txt"), "pYPKa.gb" : read_data_file("pYPKa.gb"), "pYPKpw.gb" : read_data_file("pYPKpw.gb"), "tp_g_tp.png" : read_bin_file("tp_g_tp.png"), "pYPK_ZE.png" : read_bin_file("pYPK_ZE.png"), "pYPK_A.png" : read_bin_file("pYPK_A.png"), "pw.png" : read_bin_file("pw.png"), "start.bat" : read_data_file("start.bat"), "start.sh" : read_data_file("start.sh"),} cas_vectors = "" tp_gene_tp_links = "" pYPKa_clones="" pwname = "pYPK0" genes = 0 nbflag=False while pth: genes+=1 first = pth.pop(0) # is sequence a tp-gene-tp vector? if cloned(pYPK0, (ZraI, EcoRV), first): m = re_cas.search(first.description) if not m: raise Exception( "{} is a pYPK0 tp-gene_tp sequence but was not correctly named.".format(last.description)) fn = first.description+".gb" files[fn] = first.format("gb") cas_vectors+= fn+"\n" tp_gene_tp_links+= "\n[{}]({})\n".format( first.description, fn ) tp1_description = m.group(1) gene_description = m.group(2) tp2_description = m.group(3) genes+=1 else: try: middle = pth.pop(0) last = pth.pop(0) except IndexError: raise Exception("not enough sequences") prom, gene, term = first, middle, last if cloned(pYPKa, ZraI, prom): m = re_Z.search(prom.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) prom_description = m.group(1) files[m.group(0)+".gb"] = prom.format("gb") else: #print("Z"+str(files.has_key("pYPKa_ZE_{}.md".format(prom.id)))+prom.id) if "pYPKa_ZE_{}.md".format(prom.id) not in files: files[prom.id+".gb"] = prom.format("gb") nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") files["pYPKa_ZE_{}.md".format(prom.id)] = nbtemp.format(tp=prom.id) pYPKa_clones+="[pYPKa_ZE_{n}](pYPKa_ZE_{n}.ipynb) \n".format(n=prom.id) prom_description = prom.id if cloned(pYPKa, AjiI, gene): m = re_A.search(gene.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) gene_description = m.group(1) files[m.group(0)+".gb"] = gene.format("gb") if not pYPKa_A: nbflag=True else: n = "pYPKa_A_{}".format(gene.locus) files[gene.locus+".gb"] = gene.format("gb") if pYPKa_A: nbtemp = read_data_file("nb_template_pYPKa_A_insert.md") files[n+".md"] = nbtemp.format(insert=gene.locus) gene_description = gene.locus pYPKa_clones+="[{}]({}.ipynb) \n".format(n, n) else: gene_description = gene.locus if cloned(pYPKa, EcoRV, term): m = re_E.search(term.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) term_description = m.group(1) files[m.group(0)+".gb"] = term.format("gb") else: #print("E"+str(files.has_key("pYPKa_ZE_{}.md".format(term.id)))+term.id) if "pYPKa_ZE_{}.md".format(term.id) not in files: files[term.id+".gb"] = term.format("gb") nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") files["pYPKa_ZE_{}.md".format(term.id)] = nbtemp.format(tp=term.id) pYPKa_clones+="[pYPKa_ZE_{n}](pYPKa_ZE_{n}.ipynb) \n".format(n=term.id) term_description = term.id x = "pYPK0_{}_{}_{}".format(prom_description, gene_description, term_description) if pYPKa_A or nbflag: nbtemp = read_data_file("nb_template_pYPK0_tp_gene_tp.md") files[x+".md"] = nbtemp.format(tpz=prom_description, gene=gene_description, tpe=term_description) else: nbtemp = read_data_file("nb_template_pYPK0_tp_gene_tp_gap_repair.md") files[x+".md"] = nbtemp.format(tpz=prom_description, gene=gene.locus, tpe=term_description) nbflag=False cas_vectors+="\n"+x+".gb\n" tp_gene_tp_links+="[{}]({}.ipynb) \n".format(x, x) pwname+="_{}".format(gene_description) ########################################################################### obj = notedown.MarkdownReader() cwd = os.getcwd() try: os.makedirs(dir_) except OSError as exception: if exception.errno == errno.EEXIST: print("The {} directory already exists! Please delete or choose another name.".format(dir_)) else: print("The {} directory could not be created".format(dir_)) return None, None msg = "created subdirectory {}\n".format(dir_) print(msg) log+=msg os.chdir(dir_) msg = "\nsaving files sequence files and images..\n" print(msg) log+=msg for name, content in sorted((n, c) for n, c in list(files.items()) if not n.endswith(".md")): msg = "\nsaving: "+name print(msg) log+=msg mode = {True:"wb", False:"w"}[hasattr(content, "decode")] with open(name, mode) as f: #with open(name,"wb") as f: f.write(content) print("\n") log+="\n" msg = "\nsaving notebook files ..\n" print(msg) log+=msg for name, content in sorted((n, c) for n, c in list(files.items()) if n.endswith(".md")): newname = os.path.splitext(name)[0]+".ipynb" msg = "\nsaving: "+newname print(msg) log+=msg nb = nbformat.write(obj.to_notebook(content), newname) pp = ExecutePreprocessor() pp.timeout = 120 # seconds pp.interrupt_on_timeout = True print("\n") log+="\n" msg = "\nexecuting pYPKa notebooks..\n" print(msg) log+=msg shell = InteractiveShell.instance() #new_primers = [] g={} l={} pypkanbs = sorted([f for f in os.listdir(".") if re.match("pYPKa.+\.ipynb", f)]) if pypkanbs: for name in pypkanbs: msg = "\nexecuting: "+name print(msg) log+=msg with io.open(name, 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) #new_primers.extend( (l["fp"], l["rp"]) ) nbformat.write(nb, name) g={} l={} else: msg = "\nNo pYPKa notebooks found.\n" print(msg) log+=msg print("\n") log+="\n" msg = "\nexecuting pYPK0 notebooks..\n" print(msg) log+=msg g={} l={} resources={} pypk0nbs = sorted([f for f in os.listdir(".") if re.match("pYPK0.+\.ipynb", f)]) if pypk0nbs: for name in pypk0nbs: msg = "\nexecuting: "+name print(msg) log+=msg with io.open(name, 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) nbformat.write(nb, name) for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) #try: #new_primers.extend( (l["fp"], l["rp"]) ) #except KeyError: # pass g={} l={} else: msg = "\nNo pYPK0 notebooks found.\n" print(msg) log+=msg nbtemp = read_data_file("nb_template_pYPK0_pw.md") #primer_list = "\n".join( p.format("tab") for p in new_primers ) #if new_primers: # msg = u"\n\nsaving new_primers.txt..\n" #with open("new_primers.txt","wb") as f: f.write("\n".join( p.format("fasta") for p in new_primers )) #print("qwerty") #print(pwname) #print(os.path.basename(dir_)) #print(tp_gene_tp_links) #print(add_space(cas_vectors, 17)) #print(pYPKa_clones) #print(str(genes)) #print("123456789") pwnb = nbtemp.format(name=pwname, filename=os.path.basename(dir_), tp_gene_tp_links = tp_gene_tp_links, cas_vectors=add_space(cas_vectors, 17), pYPKa_clones=pYPKa_clones, length=genes) nb = nbformat.write(obj.to_notebook(pwnb), "pw.ipynb") #nb = nbformat.writes("pw.ipynb", obj.to_notebook(pwnb)) #with open("pw.ipynb", "w") as f: f.write(nb) msg = "\n\nexecuting final pathway notebook..\n" print(msg) log+=msg msg = "\nexecuting: pw.ipynb" print(msg) log+=msg with io.open("pw.ipynb", 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) nbformat.write(nb, "pw.ipynb") #for nb_ in [f for f in os.listdir(".") if f.endswith(".ipynb")]: # subprocess.Popen(["ipython", "nbconvert", os.path.join(dir_, nb_)]) os.chdir(cwd) fl = FileLink(os.path.join(dir_, "pw.ipynb")) # pp = None return fl, log
nb_style # In[7]: from pathlib import Path from pidgin import markdown, conventions from nbconvert.exporters.python import PythonExporter from IPython.utils.capture import capture_output from nbconvert.preprocessors.execute import ExecutePreprocessor # In[8]: exporter = PythonExporter(preprocessors=[ markdown.Normalize(), conventions.Normalize(), ExecutePreprocessor() ]) def test_nbconvert_script(): with capture_output(): Path('informal_script.py').write_text( exporter.from_filename(__file__)[0]) import informal_script assert informal_script, """The script was not created.""" print('nbconvert is complete.') (__name__ == '__main__' and Path(__file__).parts[-1].startswith('informal.')) and test_nbconvert_script()
def run_notebook(self, name, event=None, timeout=None): """ run a given notebook immediately. Args: name (str): the name of the jobfile event (str): an event name timeout (int): timeout in seconds Returns: Metadata of results See Also: * nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute kwargs # -- see ExecuteProcessor class # -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html ep_kwargs = { # avoid timeouts to stop kernel 'timeout': timeout, # avoid kernel at exit functions # -- this stops ipykernel AttributeError 'send_multipart' 'shutdown_kernel': 'immediate', # set kernel name, blank is default # -- e.g. python3, ir # -- see https://stackoverflow.com/a/47053020/890242 'kernel_name': '', } # other interesting options ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: resources = { 'metadata': { 'path': self.defaults.OMEGA_TMP, } } if not meta_job.kind_meta.get('keep_output', False): # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, resources) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' finally: del ep # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results
import os import sys from nbconvert.preprocessors.execute import ExecutePreprocessor from nbconvert.nbconvertapp import NbConvertApp from nbconvert.preprocessors import ExecutePreprocessor from nbconvert.preprocessors.execute import CellExecutionError import nbformat folder = 'notebooks' os.chdir(folder) nb_list = [fname for fname in os.listdir('.') if fname.endswith('.ipynb')] nb_list.sort() ex = ExecutePreprocessor() ex.timeout = 180 # seconds ex.interrupt_on_timeout = True has_error = False for notebook in nb_list: nb = nbformat.read(notebook, as_version=4) try: nb_executed, resources = ex.preprocess(nb, resources={}) except CellExecutionError as e: print('Fail: %s \n%s\n\n' % (notebook, e.traceback[-1])) has_error = True os.chdir('..') sys.exit(-1 if has_error else 0)