def run_notebook(self, name): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now().strftime('%s') # execute try: ep = ExecutePreprocessor() ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = str(e) else: status = 'OK' # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', {}) job_runs[ts] = status meta_job.attributes['job_runs'] = job_runs meta_job.save() return meta_job
def test_demo(self): import nbformat from nbconvert.preprocessors.execute import ExecutePreprocessor here = os.path.dirname(__file__) demo = os.path.join(here,'Demo.ipynb') nb = nbformat.read(demo, as_version=4) pp = ExecutePreprocessor() pp.allow_errors = False pp.preprocess(nb, resources={})
def run_notebook(self, name, event=None): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute ep_kwargs = {'timeout': None} ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: if not meta_job.kind_meta.get('keep_output', False): resources = { } # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results
def run_notebook(self, name, event=None): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute try: ep = ExecutePreprocessor() ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' # record results meta_results = self.put( notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts return meta_job.save()
def md2ipynb(): assert len(sys.argv) == 3, 'usage: input.md output.rst' (src_fn, input_fn, output_fn) = sys.argv # timeout for each notebook, in sec timeout = 20 * 60 # if enable evaluation do_eval = int(os.environ.get('EVAL', True)) reader = notedown.MarkdownReader() with open(input_fn, 'r') as f: notebook = reader.read(f) notebook['metadata'].update({'language_info': { 'name': 'R' }}) # need to add language info for syntax highlight if do_eval: tic = time.time() executor = ExecutePreprocessor(timeout=timeout, kernel_name='ir') print('%s: Evaluated %s in %f sec' % (src_fn, input_fn, time.time() - tic)) try: notebook, resources = executor.preprocess(notebook, resources={}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % input_fn msg += 'See notebook "%s" for the traceback.' % output_fn print(msg) raise finally: with open(output_fn, 'w') as f: f.write(nbformat.writes(notebook).encode('utf8')) print('%s: Write results into %s' % (src_fn, output_fn))
def pYPKa_ZE_ipynb_generator(tp, dir_="pYPKa_ZE_vectors"): cwd = os.getcwd() try: os.makedirs(dir_) except OSError as exception: if exception.errno == errno.EEXIST: pass else: print("The {} directory could not be created".format(dir_)) return None os.chdir(dir_) with open("standard_primers.txt","w") as f: f.write(read_data_file("standard_primers.txt")) with open("pYPKa.gb","w") as f: f.write(read_data_file("pYPKa.gb")) with open("pYPK_ZE.png","w") as f: f.write(read_bin_file("pYPK_ZE.png")) with open(tp.id+".gb","w") as f: f.write(tp.format("gb")) nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") name = "pYPKa_ZE_{}.ipynb".format(tp.id) obj = notedown.MarkdownReader() nb = obj.to_notebook(nbtemp.format(tp=tp.id)) pp = ExecutePreprocessor() pp.timeout = 120 # seconds pp.interrupt_on_timeout = True shell = InteractiveShell.instance() nb_executed, resources = pp.preprocess(nb, resources={}) g={} l={} from io import StringIO old_stdout = sys.stdout redirected_output = sys.stdout = StringIO() for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) sys.stdout = old_stdout nbformat.write(nb, name) os.chdir(cwd) return FileLinks(dir_)
def execute_notebook(notebook): # Configure the notebook executor and then run the notebook # given, returning the results to the call c = Config() c.ScriptExporter.preprocessors = [NarrativePreprocessor] nar_templates = os.path.join(os.environ.get('NARRATIVE_DIR', '.'), 'src', 'biokbase', 'narrative', 'exporter', 'templates') c.TemplateExporter.template_path = ['.', nar_templates] # Initialize the notebook execution object, and run the notebook. If a # timeout (in seconds) is defined in KB_CELL_TIMEOUT, use that for # how long we allow a cell to run before timing it out, otherwise use # a default value of 60 minutes. # /tmp is the directory where the notebook will be run. if 'KB_CELL_TIMEOUT' in os.environ: timeout = int(os.environ['KB_CELL_TIMEOUT']) else: timeout = 3600 ep = ExecutePreprocessor(timeout=timeout) resources = {'metadata': {'path': '/tmp'}} return (ep.preprocess(notebook, resources))
finalcseguidZ=finalcseguidZ, finalcseguidE=finalcseguidE, fpn=fp.name, fps=fp.seq, rpn=rp.name, rps=rp.seq) obj = notedown.MarkdownReader() nb = obj.to_notebook(content) pp = ExecutePreprocessor(timeout=600, kernel_name='python3') pp.timeout = 120 # seconds pp.interrupt_on_timeout = True pp.preprocess(nb, resources={}) with open(newname, 'wt') as f: nbformat.write(nb, f) #os.chdir(cwd) # with open("README_template.md", "r", encoding="utf8") as f: # t=f.read() # table = "| No. | TP | Promoter vector | Terminator vector | Jupyter nb |\n" # table+= "|-----|----|-----------------|-------------------|------------|\n" # for no, ins in enumerate(sorted(tp_dict)): # fld = "pYPKa_ZE" # row = f"|{no} |{ins} | [pYPKa_Z_{ins}.gb]({fld}/pYPKa_Z_{ins}.gb) | [pYPKa_E_{ins}.gb]({fld}/pYPKa_E_{ins}.gb) | [nb]({fld}/pYPKa_ZE_{ins}.ipynb) |\n"
def run(notebook, timeout=30): executor = ExecutePreprocessor(timeout=timeout) notebook, resources = executor.preprocess(notebook, resources={})
def run(notebook, timeout=30, store_widget_state=True): executor = ExecutePreprocessor(timeout=timeout, store_widget_state=store_widget_state) notebook, resources = executor.preprocess(notebook, resources={})
def run_code(source: tuple, kernel_name='python3'): """Run code blocks inside a jupyter kernel. Args: source (tuple): source code blocks kernel_name: name of the kernel (form the kernel spec) to be created """ import IPython if pkg_version.parse(IPython.__version__) < pkg_version.parse('7.6.0'): raise RuntimeError("IPython version {} not supported." " Kale requires at least version 7.6.0.".format( IPython.__version__)) # new notebook spec = get_kernel_spec(kernel_name) notebook = nbformat.v4.new_notebook( metadata={ 'kernelspec': { 'display_name': spec.display_name, 'language': spec.language, 'name': kernel_name, } }) notebook.cells = [nbformat.v4.new_code_cell(s) for s in source] # these parameters are passed to nbconvert.ExecutePreprocessor jupyter_execute_kwargs = dict(timeout=-1, allow_errors=True, store_widget_state=True) resources = {} # cwd: If supplied, the kernel will run in this directory # resources['metadata'] = {'path': cwd} ep = ExecutePreprocessor(**jupyter_execute_kwargs) km = ep.kernel_manager_class(kernel_name=kernel_name, config=ep.config) # start_kernel supports several additional arguments via **kw km.start_kernel(extra_arguments=ep.extra_arguments) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=60) except RuntimeError: kc.stop_channels() raise kc.allow_stdin = False def signal_handler(_signal, _frame): raise KaleKernelException() # this signal is used by the thread in case an error message is received # by the kernel. Running sys.exit() inside the thread would terminate # just the thread itself, not the main process. Calling os._exit() can be # dangerous as the process is killed instantly (files and connections are # not closed, for example). With a signal we can capture the ExitCommand # exception from the main process and exit gracefully. signal.signal(signal.SIGUSR1, signal_handler) # start separate thread in to capture and print stdout, stderr, errors. # daemon mode will make the watcher thread die when the main one returns. x = threading.Thread(target=capture_streams, args=( kc, True, ), daemon=True) x.start() try: # start preprocessor: run each code cell and capture the output ep.preprocess(notebook, resources, km=km) except KaleKernelException: # exit gracefully with error sys.exit(-1) # Give some time to the stream watcher thread to receive all messages from # the kernel before shutting down. time.sleep(1) km.shutdown_kernel() result = process_outputs(notebook.cells) return result
def pathway(pth, dir_="ypkassembly", pYPKa_A=True, print=print): if len(pth)==0: # pth has to contain some sequences print("No of sequences found.") return None, None names = [s.name for s in pth] # sequence names has to be unique #if len(names)>len(set(names)): # print("Gene names are not unique. Please rename sequences so that each sequence has a unique name.\n") # print("Gene names parsed from Data page:\n\n") # for name in names: # print(name) # return None, None log="" pYPK0 = read(read_data_file("pYPK0.gb")) pYPKa = read(read_data_file("pYPKa.gb")) from Bio.Restriction import ZraI, AjiI, EcoRV files = {"standard_primers.txt" : read_data_file("standard_primers.txt"), "pYPKa.gb" : read_data_file("pYPKa.gb"), "pYPKpw.gb" : read_data_file("pYPKpw.gb"), "tp_g_tp.png" : read_bin_file("tp_g_tp.png"), "pYPK_ZE.png" : read_bin_file("pYPK_ZE.png"), "pYPK_A.png" : read_bin_file("pYPK_A.png"), "pw.png" : read_bin_file("pw.png"), "start.bat" : read_data_file("start.bat"), "start.sh" : read_data_file("start.sh"),} cas_vectors = "" tp_gene_tp_links = "" pYPKa_clones="" pwname = "pYPK0" genes = 0 nbflag=False while pth: genes+=1 first = pth.pop(0) # is sequence a tp-gene-tp vector? if cloned(pYPK0, (ZraI, EcoRV), first): m = re_cas.search(first.description) if not m: raise Exception( "{} is a pYPK0 tp-gene_tp sequence but was not correctly named.".format(last.description)) fn = first.description+".gb" files[fn] = first.format("gb") cas_vectors+= fn+"\n" tp_gene_tp_links+= "\n[{}]({})\n".format( first.description, fn ) tp1_description = m.group(1) gene_description = m.group(2) tp2_description = m.group(3) genes+=1 else: try: middle = pth.pop(0) last = pth.pop(0) except IndexError: raise Exception("not enough sequences") prom, gene, term = first, middle, last if cloned(pYPKa, ZraI, prom): m = re_Z.search(prom.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) prom_description = m.group(1) files[m.group(0)+".gb"] = prom.format("gb") else: #print("Z"+str(files.has_key("pYPKa_ZE_{}.md".format(prom.id)))+prom.id) if "pYPKa_ZE_{}.md".format(prom.id) not in files: files[prom.id+".gb"] = prom.format("gb") nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") files["pYPKa_ZE_{}.md".format(prom.id)] = nbtemp.format(tp=prom.id) pYPKa_clones+="[pYPKa_ZE_{n}](pYPKa_ZE_{n}.ipynb) \n".format(n=prom.id) prom_description = prom.id if cloned(pYPKa, AjiI, gene): m = re_A.search(gene.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) gene_description = m.group(1) files[m.group(0)+".gb"] = gene.format("gb") if not pYPKa_A: nbflag=True else: n = "pYPKa_A_{}".format(gene.locus) files[gene.locus+".gb"] = gene.format("gb") if pYPKa_A: nbtemp = read_data_file("nb_template_pYPKa_A_insert.md") files[n+".md"] = nbtemp.format(insert=gene.locus) gene_description = gene.locus pYPKa_clones+="[{}]({}.ipynb) \n".format(n, n) else: gene_description = gene.locus if cloned(pYPKa, EcoRV, term): m = re_E.search(term.description) if not m: raise Exception( "{} is a pYPKa_A_gene sequence but was incorrectly named.".format(gene.description)) term_description = m.group(1) files[m.group(0)+".gb"] = term.format("gb") else: #print("E"+str(files.has_key("pYPKa_ZE_{}.md".format(term.id)))+term.id) if "pYPKa_ZE_{}.md".format(term.id) not in files: files[term.id+".gb"] = term.format("gb") nbtemp = read_data_file("nb_template_pYPKa_ZE_insert.md") files["pYPKa_ZE_{}.md".format(term.id)] = nbtemp.format(tp=term.id) pYPKa_clones+="[pYPKa_ZE_{n}](pYPKa_ZE_{n}.ipynb) \n".format(n=term.id) term_description = term.id x = "pYPK0_{}_{}_{}".format(prom_description, gene_description, term_description) if pYPKa_A or nbflag: nbtemp = read_data_file("nb_template_pYPK0_tp_gene_tp.md") files[x+".md"] = nbtemp.format(tpz=prom_description, gene=gene_description, tpe=term_description) else: nbtemp = read_data_file("nb_template_pYPK0_tp_gene_tp_gap_repair.md") files[x+".md"] = nbtemp.format(tpz=prom_description, gene=gene.locus, tpe=term_description) nbflag=False cas_vectors+="\n"+x+".gb\n" tp_gene_tp_links+="[{}]({}.ipynb) \n".format(x, x) pwname+="_{}".format(gene_description) ########################################################################### obj = notedown.MarkdownReader() cwd = os.getcwd() try: os.makedirs(dir_) except OSError as exception: if exception.errno == errno.EEXIST: print("The {} directory already exists! Please delete or choose another name.".format(dir_)) else: print("The {} directory could not be created".format(dir_)) return None, None msg = "created subdirectory {}\n".format(dir_) print(msg) log+=msg os.chdir(dir_) msg = "\nsaving files sequence files and images..\n" print(msg) log+=msg for name, content in sorted((n, c) for n, c in list(files.items()) if not n.endswith(".md")): msg = "\nsaving: "+name print(msg) log+=msg mode = {True:"wb", False:"w"}[hasattr(content, "decode")] with open(name, mode) as f: #with open(name,"wb") as f: f.write(content) print("\n") log+="\n" msg = "\nsaving notebook files ..\n" print(msg) log+=msg for name, content in sorted((n, c) for n, c in list(files.items()) if n.endswith(".md")): newname = os.path.splitext(name)[0]+".ipynb" msg = "\nsaving: "+newname print(msg) log+=msg nb = nbformat.write(obj.to_notebook(content), newname) pp = ExecutePreprocessor() pp.timeout = 120 # seconds pp.interrupt_on_timeout = True print("\n") log+="\n" msg = "\nexecuting pYPKa notebooks..\n" print(msg) log+=msg shell = InteractiveShell.instance() #new_primers = [] g={} l={} pypkanbs = sorted([f for f in os.listdir(".") if re.match("pYPKa.+\.ipynb", f)]) if pypkanbs: for name in pypkanbs: msg = "\nexecuting: "+name print(msg) log+=msg with io.open(name, 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) #new_primers.extend( (l["fp"], l["rp"]) ) nbformat.write(nb, name) g={} l={} else: msg = "\nNo pYPKa notebooks found.\n" print(msg) log+=msg print("\n") log+="\n" msg = "\nexecuting pYPK0 notebooks..\n" print(msg) log+=msg g={} l={} resources={} pypk0nbs = sorted([f for f in os.listdir(".") if re.match("pYPK0.+\.ipynb", f)]) if pypk0nbs: for name in pypk0nbs: msg = "\nexecuting: "+name print(msg) log+=msg with io.open(name, 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) nbformat.write(nb, name) for cell in nb.cells: if cell.cell_type == 'code': code = shell.input_transformer_manager.transform_cell(cell.source) exec(code, g, l) #try: #new_primers.extend( (l["fp"], l["rp"]) ) #except KeyError: # pass g={} l={} else: msg = "\nNo pYPK0 notebooks found.\n" print(msg) log+=msg nbtemp = read_data_file("nb_template_pYPK0_pw.md") #primer_list = "\n".join( p.format("tab") for p in new_primers ) #if new_primers: # msg = u"\n\nsaving new_primers.txt..\n" #with open("new_primers.txt","wb") as f: f.write("\n".join( p.format("fasta") for p in new_primers )) #print("qwerty") #print(pwname) #print(os.path.basename(dir_)) #print(tp_gene_tp_links) #print(add_space(cas_vectors, 17)) #print(pYPKa_clones) #print(str(genes)) #print("123456789") pwnb = nbtemp.format(name=pwname, filename=os.path.basename(dir_), tp_gene_tp_links = tp_gene_tp_links, cas_vectors=add_space(cas_vectors, 17), pYPKa_clones=pYPKa_clones, length=genes) nb = nbformat.write(obj.to_notebook(pwnb), "pw.ipynb") #nb = nbformat.writes("pw.ipynb", obj.to_notebook(pwnb)) #with open("pw.ipynb", "w") as f: f.write(nb) msg = "\n\nexecuting final pathway notebook..\n" print(msg) log+=msg msg = "\nexecuting: pw.ipynb" print(msg) log+=msg with io.open("pw.ipynb", 'r', encoding='utf-8') as f: nb = nbformat.read(f, 4) nb_executed, resources = pp.preprocess(nb, resources={}) nbformat.write(nb, "pw.ipynb") #for nb_ in [f for f in os.listdir(".") if f.endswith(".ipynb")]: # subprocess.Popen(["ipython", "nbconvert", os.path.join(dir_, nb_)]) os.chdir(cwd) fl = FileLink(os.path.join(dir_, "pw.ipynb")) # pp = None return fl, log
#!/usr/bin/env python #-*-coding:utf-8-*- import sys import os import nbformat from nbconvert.preprocessors.execute import ExecutePreprocessor input_fn = sys.argv[1] output_fn = sys.argv[2] timeout = 60 * 20 notebook = nbformat.read(input_fn, as_version=4) executor = ExecutePreprocessor(timeout=timeout) notebook, resources = executor.preprocess(notebook, resources={}) nbformat.write(notebook, output_fn)
def run_notebook(self, name, event=None, timeout=None): """ run a given notebook immediately. Args: name (str): the name of the jobfile event (str): an event name timeout (int): timeout in seconds Returns: Metadata of results See Also: * nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute kwargs # -- see ExecuteProcessor class # -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html ep_kwargs = { # avoid timeouts to stop kernel 'timeout': timeout, # avoid kernel at exit functions # -- this stops ipykernel AttributeError 'send_multipart' 'shutdown_kernel': 'immediate', # set kernel name, blank is default # -- e.g. python3, ir # -- see https://stackoverflow.com/a/47053020/890242 'kernel_name': '', } # other interesting options ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: resources = { 'metadata': { 'path': self.defaults.OMEGA_TMP, } } if not meta_job.kind_meta.get('keep_output', False): # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, resources) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' finally: del ep # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results
import os import sys from nbconvert.preprocessors.execute import ExecutePreprocessor from nbconvert.nbconvertapp import NbConvertApp from nbconvert.preprocessors import ExecutePreprocessor from nbconvert.preprocessors.execute import CellExecutionError import nbformat folder = 'notebooks' os.chdir(folder) nb_list = [fname for fname in os.listdir('.') if fname.endswith('.ipynb')] nb_list.sort() ex = ExecutePreprocessor() ex.timeout = 180 # seconds ex.interrupt_on_timeout = True has_error = False for notebook in nb_list: nb = nbformat.read(notebook, as_version=4) try: nb_executed, resources = ex.preprocess(nb, resources={}) except CellExecutionError as e: print('Fail: %s \n%s\n\n' % (notebook, e.traceback[-1])) has_error = True os.chdir('..') sys.exit(-1 if has_error else 0)