def simpleTest2():
wf = PypeWorkflow()
f1 = makePypeLocalFile("test.fa")
f2 = makePypeLocalFile("ref.fa")
f3 = makePypeLocalFile("aln.txt", readOnly=False)
f4 = makePypeLocalFile("aln2.txt", readOnly=False)
os.system("touch %s" % f1.localFileName)
os.system("touch %s" % f2.localFileName)
@PypeTask(inputDataObjs={
"fasta": f1,
"ref": f2
},
outputDataObjs={"aln": f3},
parameters={"a": 10},
**{"b": 12})
def testTask(*argv, **kwargv):
print("testTask is running")
for ft, f in testTask.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
runShellCmd(["sleep", "5"])
@PypeTask(inputDataObjs={
"fasta": f1,
"aln": f3
},
outputDataObjs={"aln2": f4},
parameters={"a": 10},
**{"b": 12})
def testTask2(*argv, **kwargv):
print("testTask2 is running")
for ft, f in testTask2.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
#wf.addObjects([f1,f2,f3,f4]) wf.addObjects([testTask, testTask2])
wf.addTasks([testTask, testTask2])
print(wf.RDFXML)
print(wf.graphvizDot)
#aGraph = PypeGraph(wf._RDFGraph) print(aGraph.tSort())
wf.refreshTargets([f4])
print("re-touch f1")
os.system("sleep 1;touch %s;" % f1.localFileName)
wf.refreshTargets([f4])
print("re-touch f3")
os.system("sleep 1;touch %s;" % f3.localFileName)
def simpleTest2():
wf = PypeWorkflow()
f1 = makePypeLocalFile("test.fa")
f2 = makePypeLocalFile("ref.fa")
f3 = makePypeLocalFile("aln.txt", readOnly=False)
f4 = makePypeLocalFile("aln2.txt", readOnly=False)
os.system("touch %s" % f1.localFileName)
os.system("touch %s" % f2.localFileName)
@PypeTask(inputDataObjs={"fasta":f1, "ref":f2},
outputDataObjs={"aln":f3},
parameters={"a":10}, **{"b":12})
def testTask(*argv, **kwargv):
print("testTask is running")
for ft, f in testTask.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
runShellCmd(["sleep", "5" ])
@PypeTask(inputDataObjs={"fasta":f1, "aln":f3},
outputDataObjs={"aln2":f4},
parameters={"a":10}, **{"b":12})
def testTask2(*argv, **kwargv):
print("testTask2 is running")
for ft, f in testTask2.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
#wf.addObjects([f1,f2,f3,f4]) wf.addObjects([testTask, testTask2])
wf.addTasks([testTask, testTask2])
print (wf.RDFXML)
print (wf.graphvizDot)
#aGraph = PypeGraph(wf._RDFGraph) print(aGraph.tSort())
wf.refreshTargets([f4])
print("re-touch f1")
os.system("sleep 1;touch %s;" % f1.localFileName)
wf.refreshTargets([f4])
print("re-touch f3")
os.system("sleep 1;touch %s;" % f3.localFileName)
def simpleTest():
wf = PypeWorkflow()
# f1 and f2 are the mock input files
f1 = makePypeLocalFile("test.fa")
f2 = makePypeLocalFile("ref.fa")
# f3 is the object of the expected output of the "testTask"
f3 = makePypeLocalFile("aln.txt", readOnly=False)
# create the mock files
os.system("touch %s" % f1.localFileName)
os.system("touch %s" % f2.localFileName)
# the testTask will take f1 (as "testTask.fasta") and f2 (as "testTask.ref") and generate f3 (as "testTask.aln")
@PypeTask(inputDataObjs={"fasta":f1, "ref":f2},
outputDataObjs={"aln":f3},
parameters={"a":10}, **{"b":12})
def testTask(*argv, **kwargv):
print("testTask is running")
print("fasta input filename is %s" % testTask.fasta.localFileName)
for ft, f in testTask.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
runShellCmd(["sleep", "5" ])
# the testTask will take f1 (as "testTask.fasta") and f3 (as "testTask.aln") and generate f4 (as "testTask.aln2")
f4 = makePypeLocalFile("aln2.txt", readOnly=False)
@PypeTask(inputDataObjs={"fasta":f1, "aln":f3},
outputDataObjs={"aln2":f4},
parameters={"a":10}, **{"b":12})
def testTask2(*argv, **kwargv):
print("testTask2 is running")
for ft, f in testTask2.outputDataObjs.iteritems():
#os.system("touch %s" % f.localFileName)
runShellCmd(["touch", "%s" % f.localFileName])
# one can add objects one by one to the workflow
#wf.addObjects([f1,f2,f3,f4])
#wf.addObjects([testTask, testTask2])
# or, one can add the "tasks" into the workflow, the input and output data objects will be added automatically
wf.addTasks([testTask, testTask2])
#print out the RDFXML file that represents the workflow
print (wf.RDFXML)
#a graphviz dot for rendering the dependency graph if one
print (wf.graphvizDot)
# execute the workflow until f4 is updated
wf.refreshTargets([f4])
# mock the case that f1 is updated
print("re-touch f1")
os.system("sleep 1;touch %s;" % f1.localFileName)
wf.refreshTargets([f4])
# mock the case that f3 is updated
print("re-touch f3")
os.system("sleep 1;touch %s;" % f3.localFileName)
def run_HGAP(config):
global prepare_data
global prepare_seed_reads
global dist_map
global generate_preassemble_reads
global run_CA
global quiver_reseq
directory_for_dist_map = "dist_map"
config["install_prefix"] = sys.prefix
config["directory_for_dist_map"] = directory_for_dist_map
input_fofn_fn = config["input_fofn_fn"]
#tmpdir = config["tmpdir"]
#prepration the distribute mapping directory
#try:
#os.makedirs("%s/ec_data" % directory_for_dist_map)
#os.makedirs("/%s/ec_data" % tmpdir)
#except:
#pass
try:
os.makedirs("%s" % directory_for_dist_map)
except:
pass
try:
os.makedirs("scripts")
except:
pass
try:
os.makedirs("CA")
except:
pass
try:
os.makedirs("sge_log")
except:
pass
input_fofn = makePypeLocalFile(input_fofn_fn)
normalized_fasta = makePypeLocalFile("all_norm.fa")
seed_fasta = makePypeLocalFile("seeds.fa")
wf = PypeWorkflow()
prepare_data_task = PypeTask(inputDataObjs={"input_fofn":input_fofn},
outputDataObjs={"normalized_fasta":normalized_fasta},
config = config ) (prepare_data)
prepare_seed_reads_task = PypeTask(inputDataObjs = {"normalized_fasta":normalized_fasta},
outputDataObjs = {"seed_fasta":seed_fasta},
config = config)(prepare_seed_reads)
wf.addTasks([prepare_data_task, prepare_seed_reads_task])
m4_data_done = makePypeLocalFile("%s/m4_data_done" % directory_for_dist_map)
dist_map_task = PypeTask(inputDataObjs = {"normalized_fasta":normalized_fasta, "seed_fasta":seed_fasta},
outputDataObjs = {"m4_data_done":m4_data_done},
config = config) (dist_map)
m4filtering_done = makePypeLocalFile("%s/m4filtering_done" % directory_for_dist_map)
m4filtering_task = PypeTask(inputDataObjs = {"m4_data_done":m4_data_done},
outputDataObjs = {"m4filtering_done":m4filtering_done},
config = config) (m4_filtering)
preassembly_done = makePypeLocalFile("%s/preassembly_done" % directory_for_dist_map)
get_preassembled_reads_task = PypeTask( inputDataObjs = {"normalized_fasta" : normalized_fasta,
"seed_fasta" : seed_fasta,
"m4filtering_done" : m4filtering_done},
outputDataObjs = {"preassembly_done" : preassembly_done},
config = config ) (get_preassembled_reads)
wf.addTasks([dist_map_task, m4filtering_task, get_preassembled_reads_task])
CA_done = makePypeLocalFile("CA_done")
run_CA_task = PypeTask( inputDataObjs = {"preassembly_done" : preassembly_done},
outputDataObjs = {"CA_done": CA_done},
config = config )(run_CA)
wf.addTasks([run_CA_task])
Quiver_done = makePypeLocalFile("Quiver_done")
quiver_reseq_task = PypeTask( inputDataObjs = {"CA_done": CA_done, "input_fofn":input_fofn},
outputDataObjs = {"Quiver_done": Quiver_done},
config = config) ( quiver_reseq )
wf.addTasks([quiver_reseq_task])
if config["target"] == "all":
wf.refreshTargets([Quiver_done])
elif config["target"] == "draft_assembly":
wf.refreshTargets([CA_done])
elif config["target"] == "pre_assembly":
wf.refreshTargets([preassembly_done])
def run_HGAP(config):
global prepare_data
global prepare_seed_reads
global dist_map
global generate_preassemble_reads
global run_CA
global quiver_reseq
directory_for_dist_map = "dist_map"
config["install_prefix"] = sys.prefix
config["directory_for_dist_map"] = directory_for_dist_map
input_fofn_fn = config["input_fofn_fn"]
#tmpdir = config["tmpdir"]
#prepration the distribute mapping directory
#try:
#os.makedirs("%s/ec_data" % directory_for_dist_map)
#os.makedirs("/%s/ec_data" % tmpdir)
#except:
#pass
try:
os.makedirs("%s" % directory_for_dist_map)
except:
pass
try:
os.makedirs("scripts")
except:
pass
try:
os.makedirs("CA")
except:
pass
try:
os.makedirs("sge_log")
except:
pass
input_fofn = makePypeLocalFile(input_fofn_fn)
normalized_fasta = makePypeLocalFile("all_norm.fa")
seed_fasta = makePypeLocalFile("seeds.fa")
wf = PypeWorkflow()
prepare_data_task = PypeTask(
inputDataObjs={"input_fofn": input_fofn},
outputDataObjs={"normalized_fasta": normalized_fasta},
config=config)(prepare_data)
prepare_seed_reads_task = PypeTask(
inputDataObjs={"normalized_fasta": normalized_fasta},
outputDataObjs={"seed_fasta": seed_fasta},
config=config)(prepare_seed_reads)
wf.addTasks([prepare_data_task, prepare_seed_reads_task])
m4_data_done = makePypeLocalFile("%s/m4_data_done" %
directory_for_dist_map)
dist_map_task = PypeTask(inputDataObjs={
"normalized_fasta": normalized_fasta,
"seed_fasta": seed_fasta
},
outputDataObjs={"m4_data_done": m4_data_done},
config=config)(dist_map)
m4filtering_done = makePypeLocalFile("%s/m4filtering_done" %
directory_for_dist_map)
m4filtering_task = PypeTask(
inputDataObjs={"m4_data_done": m4_data_done},
outputDataObjs={"m4filtering_done": m4filtering_done},
config=config)(m4_filtering)
preassembly_done = makePypeLocalFile("%s/preassembly_done" %
directory_for_dist_map)
get_preassembled_reads_task = PypeTask(
inputDataObjs={
"normalized_fasta": normalized_fasta,
"seed_fasta": seed_fasta,
"m4filtering_done": m4filtering_done
},
outputDataObjs={"preassembly_done": preassembly_done},
config=config)(get_preassembled_reads)
wf.addTasks([dist_map_task, m4filtering_task, get_preassembled_reads_task])
CA_done = makePypeLocalFile("CA_done")
run_CA_task = PypeTask(
inputDataObjs={"preassembly_done": preassembly_done},
outputDataObjs={"CA_done": CA_done},
config=config)(run_CA)
wf.addTasks([run_CA_task])
Quiver_done = makePypeLocalFile("Quiver_done")
quiver_reseq_task = PypeTask(inputDataObjs={
"CA_done": CA_done,
"input_fofn": input_fofn
},
outputDataObjs={"Quiver_done": Quiver_done},
config=config)(quiver_reseq)
wf.addTasks([quiver_reseq_task])
if config["target"] == "all":
wf.refreshTargets([Quiver_done])
elif config["target"] == "draft_assembly":
wf.refreshTargets([CA_done])
elif config["target"] == "pre_assembly":
wf.refreshTargets([preassembly_done])
