def run(config, output=sys.stdout, job_type='local', report_type='text', shell='/bin/bash',
temp='.metapipe', run_now=False):
""" Create the metapipe based on the provided input. """
parser = Parser(config)
try:
command_templates = parser.consume()
except ValueError as e:
raise SyntaxError('Invalid config file. \n%s' % e)
queue_type = QUEUE_TYPES[report_type]
pipeline = Runtime(command_templates, queue_type, JOB_TYPES, job_type)
template = env.get_template('output_script.tmpl.sh')
with open(temp, 'wb') as f:
pickle.dump(pipeline, f, 2)
script = template.render(shell=shell, temp=os.path.abspath(temp))
if run_now:
output = output if output != sys.stdout else PIPELINE_ALIAS
submit_job = make_submit_job(shell, output, job_type)
submit_job.submit()
try:
f = open(output, 'w')
output = f
except TypeError:
pass
output.write(script)
f.close()
def run(config, max_jobs, output=sys.stdout, job_type='local',
report_type='text', shell='/bin/bash', temp='.metapipe', run_now=False):
""" Create the metapipe based on the provided input. """
parser = Parser(config)
try:
command_templates = parser.consume()
except ValueError as e:
raise SyntaxError('Invalid config file. \n%s' % e)
options = '\n'.join(parser.global_options)
queue_type = QUEUE_TYPES[report_type]
print(command_templates,
queue_type,
JOB_TYPES,
job_type,
max_jobs
)
pipeline = Runtime(command_templates,
queue_type,
JOB_TYPES,
job_type=job_type,
max_jobs=max_jobs
)
template = env.get_template('output_script.tmpl.sh')
with open(temp, 'wb') as f:
pickle.dump(pipeline, f, 2)
script = template.render(shell=shell,
temp=os.path.abspath(temp), options=options)
if run_now:
output = output if output != sys.stdout else PIPELINE_ALIAS
submit_job = make_submit_job(shell, output, job_type)
submit_job.submit()
try:
f = open(output, 'w')
output = f
except TypeError:
pass
output.write(script)
f.close()
""" A series of mixins for reporting. """
from datetime import datetime as dt
from metapipe.templates import env
template = env.get_template('progress-report.tmpl.html')
class BaseReportingMixin(object):
""" An abstract mixin for reporting. """
message_format = '%Y-%m-%d %H:%M:%S'
def render(self, message, progress):
""" Render the output of the report. """
pass
class HtmlReportingMixin(BaseReportingMixin):
""" A reporting mixin that writes progress to an HTML report. """
messages = []
output = 'metapipe.report.html'
def render(self, message, progress):
msg = Message(dt.strftime(dt.now(), self.message_format), message)
self.messages.insert(0, msg)
with open(self.output, 'w') as f:
f.write(self.template.render(
name=self.name,
messages=self.messages, progress=progress, jobs=sorted(self.real_jobs)))
from __future__ import print_function
import sure
from metapipe.templates import env
from .fixtures import *
template = env.get_template('output_script.tmpl.sh')
def test_make_script():
script = template.render(shell='/usr/bin/sh', temp='metapipe.script')
script.should.equal("""#! /usr/bin/sh
set -e;
python - <<END
import pickle
with open('metapipe.script', 'rb') as f:
runtime = pickle.load(f)
runtime.run()
END""")
