def get_pipeline(self,**kwargs):
session=RnaseqGlobals.get_session()
use_template=RnaseqGlobals.conf_value('use_template')
found=False
assert(kwargs['name'])
assert(kwargs['readset'])
db_pipeline=session.query(Pipeline).filter_by(name=kwargs['name']).first()
found=db_pipeline!=None
if use_template or not found: # build pipeline using template
t_pipeline=Pipeline(name=kwargs['name'], readset=kwargs['readset']).load()
if found:
if use_template: # replace existing template with newly generated one:
assert(db_pipeline.id != None)
session.delete(db_pipeline)
session.commit()
session.add(t_pipeline)
session.commit()
pipeline=t_pipeline
else:
pipeline=db_pipeline
pipeline.readset=kwargs['readset']
else: # found==False
t_pipeline.template_file() # sets pipeline.path
session.add(t_pipeline)
session.commit()
pipeline=t_pipeline
assert(hasattr(pipeline,'readset'))
pipeline.set_defaults()
return pipeline
def setUp(self):
templated.template_dir=os.path.normpath(os.path.abspath(__file__)+"/../../fixtures/templates")
RnaseqGlobals.initialize(__file__, testing=True)
readset_file=os.path.join(RnaseqGlobals.root_dir(),'t','fixtures','readsets','readset1.syml')
self.readset=Readset.load(readset_file)[0]
self.pipeline=Pipeline(name='juan', readset=self.readset).load_steps()
session=RnaseqGlobals.get_session()
ps=session.query(Pipeline).all()
for p in ps:
session.delete(p)
session.commit()
def setUp(self):
RnaseqGlobals.initialize(__file__, testing=True)
templated.template_dir=RnaseqGlobals.root_dir()+"/t/fixtures/templates"
readset_file=RnaseqGlobals.root_dir()+"/t/fixtures/readsets/readset1.syml"
self.readset=Readset.load(readset_file)[0]
self.pipeline=Pipeline(name='juan', readset=self.readset).load_steps()
session=RnaseqGlobals.get_session()
# delete all pre-existing pipeline objects from the db:
plist=session.query(Pipeline)
for p in plist:
session.delete(p)
session.commit()
def test_query(self):
session=RnaseqGlobals.get_session()
readset=self.readset
session.add(self.pipeline)
session.commit()
pipeline=self.pipeline
l=session.query(Pipeline).filter_by(name=pipeline.name).all()
self.assertEqual(len(l), 1)
pl=l[0]
for a in ['name', 'description']:
self.assertEqual(getattr(pl,a), getattr(pipeline,a))
self.assertEqual(pl.id,1)
def run(self, *argv, **args):
session = RnaseqGlobals.get_session()
for klass in self.classes:
# Drop the table first. Don't try to use Table.drop(), it's a pain if the table doesn't already exist.
# engine.execute("DROP TABLE IF EXISTS %s" % tablename)
try:
ct = getattr(klass, "create_table")
except AttributeError as ae:
raise UserError("%s doesn't define 'create_table'" % klass.__name__)
ct(RnaseqGlobals.metadata, RnaseqGlobals.engine)
print "%s created" % klass.__tablename__
def test_pipeline_id(self):
pipeline=self.pipeline
try: id=pipeline.id
except AttributeError: self.assertTrue(True)
session=RnaseqGlobals.get_session()
session.add(pipeline)
session.commit()
session.flush()
self.assertEqual(pipeline.id, 1)
mpipeline=session.merge(pipeline)
self.assertEqual(mpipeline.id, 1)
def store_db(self):
session=RnaseqGlobals.get_session()
other_self=session.query(Pipeline).filter_by(name=self.name).first()
if other_self==None:
session.add(self)
session.commit()
#if RnaseqGlobals.conf_value('debug'): warn("s_db: added pipeline %s: id=%d" % (self.name, self.id))
#if RnaseqGlobals.conf_value('debug'): warn("s_db: %s.path: %s" % (self.name, self.path))
else:
#if RnaseqGlobals.conf_value('debug'): warn("s_db: found pipeline %s: id=%d" % (self.name, other_self.id))
self.id=other_self.id
self=other_self
return self
def setUp(self):
RnaseqGlobals.initialize(__file__, testing=True)
templated.template_dir=RnaseqGlobals.root_dir()+'/t/fixtures/templates'
os.chdir(RnaseqGlobals.root_dir()+'/t/fixtures/readsets')
filename=RnaseqGlobals.root_dir()+'/t/fixtures/readsets/readset_rel_glob.syml'
rlist=Readset.load(filename)
readset=rlist[0]
self.readset=readset
self.session=RnaseqGlobals.get_session()
rlist=self.session.query(Readset)
for rs in rlist:
self.session.delete(rs)
self.session.commit()
def run(self, *argv, **args):
try: classname=argv[0][2] # [0] is script name, [1] is command
except IndexError: raise UserError(self.usage())
try: klass=globals()[classname]
except KeyError: raise UserError("%s: unknown class" % classname)
obj_hash={}
paired=[p for p in argv[0] if re.match("\w+=\w+", p)]
for pair in paired:
k,v=re.split("=",pair)
obj_hash[k]=v
o=klass(obj_hash)
session=RnaseqGlobals.get_session()
session.add(o)
session.commit()
def test_insert(self):
session=RnaseqGlobals.get_session()
session.add(self.pipeline)
session.commit()
def sh_script(self, **kwargs):
script="#!/bin/sh\n\n"
session=RnaseqGlobals.get_session()
verbose=RnaseqGlobals.conf_value('verbose')
# determine if provenance is desired:
try:
pipeline_run=kwargs['pipeline_run']
step_runs=kwargs['step_runs']
include_provenance=True
except KeyError:
include_provenance=False
# create auxillary steps:
if include_provenance:
step_factory=StepFactory()
pipeline_start=step_factory.new_step(self,
'pipeline_start',
pipeline_run_id=pipeline_run.id,
step_run_id=None,
next_step_run_id=self.context.step_runs[self.steps[0].name].id)
mid_step=step_factory.new_step(self, 'mid_step', pipeline_run_id=pipeline_run.id)
pipeline_end=step_factory.new_step(self, 'pipeline_end', pipelinerun_id=pipeline_run.id,
step_run_id=None, next_step_run_id=None)
script+=pipeline_start.sh_script(self.context)
# iterate through steps;
errors=[]
for step in self.steps:
try:
if step.skip:
if verbose: print "skipping step %s (already current)" % step.name
continue # in a try block in case step.skip doesn't even exist
except: # really? step.skip doesn't exist, so assume it's True???
pass
# append step.sh_script(self.context)
step_script=step.sh_script(self.context, echo_name=True)
try: step_script=step.sh_script(self.context, echo_name=True)
except Exception as e:
errors.append("%s: %s" % (step.name,str(e)))
errors.append("Exception in pipeline.sh_script(step %s): %s (%s)" % (step.name, e, type(e)))
continue
script+=step_script
script+="\n"
# insert check success step:
if include_provenance:
try: skip_check=step['skip_success_check']
except: skip_check=False
if not skip_check:
step_run=step_runs[step.name]
step_run.cmd=step_script
mid_step.stepname=step.name
mid_step.step_run_id=step_run.id
next_step=self.step_after(step.name)
if next_step:
mid_step.next_step_run_id=self.context.step_runs[next_step.name].id
else:
mid_step.next_step_run_id=0
script+=mid_step.sh_script(self.context)
if RnaseqGlobals.conf_value('verbose'):
print "step %s added" % step.name
# record finish:
if include_provenance:
pipeline_end.last_step_id=step_runs[self.steps[-1].name].id
script+=pipeline_end.sh_script(self.context)
# check for continuity and raise exception on errors:
errors.extend(self.verify_continuity(self.context))
if len(errors)>0:
raise ConfigError("\n".join(errors))
session.commit()
return script
