def run(config, output=sys.stdout, job_type='local', report_type='text', shell='/bin/bash', temp='.metapipe', run_now=False): """ Create the metapipe based on the provided input. """ parser = Parser(config) try: command_templates = parser.consume() except ValueError as e: raise SyntaxError('Invalid config file. \n%s' % e) queue_type = QUEUE_TYPES[report_type] pipeline = Runtime(command_templates, queue_type, JOB_TYPES, job_type) template = env.get_template('output_script.tmpl.sh') with open(temp, 'wb') as f: pickle.dump(pipeline, f, 2) script = template.render(shell=shell, temp=os.path.abspath(temp)) if run_now: output = output if output != sys.stdout else PIPELINE_ALIAS submit_job = make_submit_job(shell, output, job_type) submit_job.submit() try: f = open(output, 'w') output = f except TypeError: pass output.write(script) f.close()
def run(config, max_jobs, output=sys.stdout, job_type='local', report_type='text', shell='/bin/bash', temp='.metapipe', run_now=False): """ Create the metapipe based on the provided input. """ parser = Parser(config) try: command_templates = parser.consume() except ValueError as e: raise SyntaxError('Invalid config file. \n%s' % e) options = '\n'.join(parser.global_options) queue_type = QUEUE_TYPES[report_type] print(command_templates, queue_type, JOB_TYPES, job_type, max_jobs ) pipeline = Runtime(command_templates, queue_type, JOB_TYPES, job_type=job_type, max_jobs=max_jobs ) template = env.get_template('output_script.tmpl.sh') with open(temp, 'wb') as f: pickle.dump(pipeline, f, 2) script = template.render(shell=shell, temp=os.path.abspath(temp), options=options) if run_now: output = output if output != sys.stdout else PIPELINE_ALIAS submit_job = make_submit_job(shell, output, job_type) submit_job.submit() try: f = open(output, 'w') output = f except TypeError: pass output.write(script) f.close()
""" A series of mixins for reporting. """ from datetime import datetime as dt from metapipe.templates import env template = env.get_template('progress-report.tmpl.html') class BaseReportingMixin(object): """ An abstract mixin for reporting. """ message_format = '%Y-%m-%d %H:%M:%S' def render(self, message, progress): """ Render the output of the report. """ pass class HtmlReportingMixin(BaseReportingMixin): """ A reporting mixin that writes progress to an HTML report. """ messages = [] output = 'metapipe.report.html' def render(self, message, progress): msg = Message(dt.strftime(dt.now(), self.message_format), message) self.messages.insert(0, msg) with open(self.output, 'w') as f: f.write(self.template.render( name=self.name, messages=self.messages, progress=progress, jobs=sorted(self.real_jobs)))
from __future__ import print_function import sure from metapipe.templates import env from .fixtures import * template = env.get_template('output_script.tmpl.sh') def test_make_script(): script = template.render(shell='/usr/bin/sh', temp='metapipe.script') script.should.equal("""#! /usr/bin/sh set -e; python - <<END import pickle with open('metapipe.script', 'rb') as f: runtime = pickle.load(f) runtime.run() END""")