def create_polish_pickle(n_polish_chunks_in_bins, flnc_files, out_pickle):
"""
Parameters:
n_polish_chunks_in_bins -- number of ice_polish chunks in each bin
flnc_files -- full-length non-chimeric files in bins
out_pickle -- output pickle for saving PolishChunkTask objects
"""
n_bins = len(flnc_files)
assert isinstance(n_polish_chunks_in_bins, list)
assert len(n_polish_chunks_in_bins) == n_bins
log.info("Writing %s ice_polish chunk tasks to %s.",
str(sum(n_polish_chunks_in_bins)), out_pickle)
p = ChunkTasksPickle()
for i, flnc_file in enumerate(flnc_files):
log.debug("Creating %s ice_polish chunks for bin index=%s.",
str(n_polish_chunks_in_bins[i]), str(i))
cluster_out_dir = _get_cluster_out_dir(flnc_file)
for j in range(0, n_polish_chunks_in_bins[i]):
# Create Polish chunk tasks.
task_ = PolishChunkTask(cluster_bin_index=i, flnc_file=flnc_file,
cluster_out_dir=cluster_out_dir,
polish_index=j,
n_polish_chunks=n_polish_chunks_in_bins[i])
p.append(task_)
p.write(out_pickle)
log.info("Saved %s polish chunk tasks to %s.", str(sum(n_polish_chunks_in_bins)), out_pickle)
def create_partial_pickle(flnc_files, chunked_nfl_files, out_pickle):
"""
Parameters:
flnc_files -- full-length non-chimeric files in bins
chunked_nfl_files -- chunked non-chimeric files
out_pickle -- output pickle for saving PolishChunkTask objects
"""
n_bins = len(flnc_files)
n_nfl_chunks = len(chunked_nfl_files)
log.info("Writing %s ice_partial chunk tasks to %s.", str(n_bins * n_nfl_chunks), out_pickle)
p = ChunkTasksPickle()
for i, flnc_file in enumerate(flnc_files):
log.debug("Processing cluster bin index=%s.", i)
cluster_out_dir = _get_cluster_out_dir(flnc_file)
for j, nfl_file in enumerate(chunked_nfl_files):
# Create Partial chunk tasks.
task_ = PartialChunkTask(cluster_bin_index=i, flnc_file=flnc_file,
cluster_out_dir=cluster_out_dir,
nfl_file=nfl_file,
nfl_index=j, n_nfl_chunks=n_nfl_chunks)
p.append(task_)
p.write(out_pickle)
log.info("Saved %s partial chunk tasks to %s.", str(n_bins * n_nfl_chunks), out_pickle)
def resolved_tool_contract_runner(rtc):
"""Given resolved tool contract, run"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, PartialChunkTask) for task in p])
dummy_sentinel_file = rtc.task.input_files[1]
ccs_file = rtc.task.input_files[2]
nproc = rtc.task.nproc
tmp_dir = rtc.task.tmpdir_resources[0].path \
if len(rtc.task.tmpdir_resources) > 0 else None
log.info("Looking for QVs in CCS input...")
with ConsensusReadSet(ccs_file) as ds:
for bam in ds.resourceReaders():
qvs = bam.pulseFeaturesAvailable()
if qvs != set(['SubstitutionQV', 'InsertionQV', 'DeletionQV']):
log.warn("Missing QV fields from %s, will use default probabilities",
bam.filename)
ccs_file = None
break
with open(rtc.task.output_files[0], 'w') as writer:
for task in p:
log.info("Running ice_partial on cluster bin %s, nfl chunk %s/%s",
str(task.cluster_bin_index),
str(task.nfl_index), str(task.n_nfl_chunks))
task_runner(task=task, ccs_file=ccs_file, nproc=nproc, tmp_dir=tmp_dir)
writer.write("ice_partial of cluster bin %s, nfl chunk %s/%s in %s is DONE: %s\n" %
(task.cluster_bin_index, task.nfl_index, task.n_nfl_chunks,
task.cluster_out_dir, task.nfl_pickle))
def resolved_tool_contract_runner(rtc):
"""
For each cluster bin, clean up intermediate files under tmp.
"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
cluster_bin_indices = [task.cluster_bin_index for task in p]
# sanity check that Cluster indices are unique!
assert len(set(cluster_bin_indices)) == len(cluster_bin_indices)
sentinel_out = rtc.task.output_files[0]
with open(sentinel_out, 'w') as writer:
for task in p:
icef = IceFiles(prog_name="ice_cleanup",
root_dir=task.cluster_out_dir)
tmp_dir = icef.tmp_dir
log.info("Cleaning up, removing %s", tmp_dir)
writer.write("removing %s\n" % tmp_dir)
execute("rm -rf %s" % tmp_dir)
quivered_dir = icef.quivered_dir
log.info("Cleaning up, removing %s", quivered_dir)
writer.write("removing %s\n" % quivered_dir)
execute("rm -rf %s" % quivered_dir)
def resolved_tool_contract_runner(rtc):
"""resolved tool contract runner."""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, PolishChunkTask) for task in p])
dummy_sentinel_file = rtc.task.input_files[1]
subread_set = rtc.task.input_files[2]
nproc = rtc.task.nproc
tmp_dir = rtc.task.tmpdir_resources[0].path \
if len(rtc.task.tmpdir_resources) > 0 else None
with open(rtc.task.output_files[0], 'w') as writer:
for task in p:
log.info(
"Running ice_polish on cluster bin %s, polish chunk %s/%s",
str(task.cluster_bin_index), str(task.polish_index),
str(task.n_polish_chunks))
log.debug("ice_quiver root_dir is %s", task.cluster_out_dir)
log.debug("consensus_isoforms is %s", task.consensus_isoforms_file)
task_runner(task=task,
subread_set=subread_set,
nproc=nproc,
tmp_dir=tmp_dir)
writer.write(
"ice_polish of cluster bin %s, polish chunk %s/%s in %s is DONE.\n"
% (task.cluster_bin_index, task.polish_index,
task.n_polish_chunks, task.cluster_out_dir))
def resolved_tool_contract_runner(rtc):
"""Given resolved tool contract, run"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, PartialChunkTask) for task in p])
dummy_sentinel_file = rtc.task.input_files[1]
ccs_file = rtc.task.input_files[2]
nproc = rtc.task.nproc
tmp_dir = rtc.task.tmpdir_resources[0].path \
if len(rtc.task.tmpdir_resources) > 0 else None
log.info("Looking for QVs in CCS input...")
with ConsensusReadSet(ccs_file) as ds:
for bam in ds.resourceReaders():
qvs = bam.pulseFeaturesAvailable()
if qvs != set(['SubstitutionQV', 'InsertionQV', 'DeletionQV']):
log.warn(
"Missing QV fields from %s, will use default probabilities",
bam.filename)
ccs_file = None
break
with open(rtc.task.output_files[0], 'w') as writer:
for task in p:
log.info("Running ice_partial on cluster bin %s, nfl chunk %s/%s",
str(task.cluster_bin_index), str(task.nfl_index),
str(task.n_nfl_chunks))
task_runner(task=task,
ccs_file=ccs_file,
nproc=nproc,
tmp_dir=tmp_dir)
writer.write(
"ice_partial of cluster bin %s, nfl chunk %s/%s in %s is DONE: %s\n"
% (task.cluster_bin_index, task.nfl_index, task.n_nfl_chunks,
task.cluster_out_dir, task.nfl_pickle))
def resolved_tool_contract_runner(rtc):
"""
For each cluster bin, clean up intermediate files under tmp.
"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
cluster_bin_indices = [task.cluster_bin_index for task in p]
# sanity check that Cluster indices are unique!
assert len(set(cluster_bin_indices)) == len(cluster_bin_indices)
sentinel_out = rtc.task.output_files[0]
with open(sentinel_out, 'w') as writer:
for task in p:
icef = IceFiles(prog_name="ice_cleanup",
root_dir=task.cluster_out_dir)
tmp_dir = icef.tmp_dir
log.info("Cleaning up, removing %s", tmp_dir)
writer.write("removing %s\n" % tmp_dir)
execute("rm -rf %s" % real_upath(tmp_dir))
quivered_dir = icef.quivered_dir
log.info("Cleaning up, removing %s", quivered_dir)
writer.write("removing %s\n" % quivered_dir)
execute("rm -rf %s" % real_upath(quivered_dir))
def resolved_tool_contract_runner(rtc):
"""run all tasks in cluster_chunks.pickle given rtc"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
ccs_file = rtc.task.input_files[1]
assert op.exists(ccs_file)
nproc = rtc.task.nproc
use_finer_qv = False
#if rtc.task.options.get(Constants.USE_FINER_QV_ID, False):
# use_finer_qv = True
with open(rtc.task.output_files[0], 'w') as writer:
for i, task in enumerate(p):
args = task_to_args(task=task,
ccs_file=ccs_file,
nproc=nproc,
use_finer_qv=use_finer_qv)
log.info("ARGUMENTS of Task %s/%s:\n%s", str(i), str(len(p)),
str(args))
log.info("Running ICE on cluster bin %s", task.cluster_bin_index)
PBTranscript(args, subCommand="cluster").start()
writer.write("ICE of cluster bin %s in %s is DONE: %s\n" %
(task.cluster_bin_index, task.cluster_out_dir,
task.consensus_isoforms_file))
def run_main(cluster_chunks_pickle_file, ccs_file, output_json_file,
max_nchunks):
"""Scatter items in cluster_chunks_pickle
Parameters:
cluster_chunks_pickle_file -- ChunkTasksPickle of ClusterChunkTask objects.
ccs_file -- ccs.consensusreadset.xml
output_json_file -- chunk.json
max_nchunks -- maximum # of chunks
"""
p = ChunkTasksPickle.read(cluster_chunks_pickle_file)
assert all([isinstance(r, ClusterChunkTask) for r in p])
out_dir = op.dirname(output_json_file)
# sort and group tasks
groups = p.sort_and_group_tasks(max_nchunks=max_nchunks)
# Writing chunk.json
base_name = "spawned_cluster_chunk"
chunks = []
spawned_pickles = []
for group_index in range(0, len(groups)):
chunk_id = "_".join([base_name, 'group', str(group_index)])
spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
d = {
Constants.CHUNK_KEYS[0]: spawned_pickle_file,
Constants.CHUNK_KEYS[1]: ccs_file
}
c = PipelineChunk(chunk_id, **d)
chunks.append(c)
spawned_pickles.append(spawned_pickle_file)
log.info("Spawning %s into %d files", cluster_chunks_pickle_file,
len(groups))
p.spawn_pickles_by_groups(groups, spawned_pickles)
log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
# n_chunks = len(p)
# for i in range(0, n_chunks):
# chunk_id = "_".join([base_name, str(i)])
# spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
# d = {Constants.CHUNK_KEYS[0]: spawned_pickle_file,
# Constants.CHUNK_KEYS[1]: ccs_file}
# c = PipelineChunk(chunk_id, **d)
# chunks.append(c)
# spawned_pickles.append(spawned_pickle_file)
#
# log.info("Spawning %s into %s files", cluster_chunks_pickle_file, str(n_chunks))
# p.spawn_pickles(spawned_pickles)
# log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
log.info("Writing chunk.json to %s", output_json_file)
write_pipeline_chunks(chunks, output_json_file,
"created by %s" % Constants.TOOL_ID)
return 0
def create_polish_pickle(n_polish_chunks_in_bins, flnc_files, out_pickle):
"""
Parameters:
n_polish_chunks_in_bins -- number of ice_polish chunks in each bin
flnc_files -- full-length non-chimeric files in bins
out_pickle -- output pickle for saving PolishChunkTask objects
"""
n_bins = len(flnc_files)
assert isinstance(n_polish_chunks_in_bins, list)
assert len(n_polish_chunks_in_bins) == n_bins
log.info("Writing %s ice_polish chunk tasks to %s.",
str(sum(n_polish_chunks_in_bins)), out_pickle)
p = ChunkTasksPickle()
for i, flnc_file in enumerate(flnc_files):
log.debug("Creating %s ice_polish chunks for bin index=%s.",
str(n_polish_chunks_in_bins[i]), str(i))
cluster_out_dir = _get_cluster_out_dir(flnc_file)
for j in range(0, n_polish_chunks_in_bins[i]):
# Create Polish chunk tasks.
task_ = PolishChunkTask(cluster_bin_index=i,
flnc_file=flnc_file,
cluster_out_dir=cluster_out_dir,
polish_index=j,
n_polish_chunks=n_polish_chunks_in_bins[i])
p.append(task_)
p.write(out_pickle)
log.info("Saved %s polish chunk tasks to %s.",
str(sum(n_polish_chunks_in_bins)), out_pickle)
def create_partial_pickle(flnc_files, chunked_nfl_files, out_pickle):
"""
Parameters:
flnc_files -- full-length non-chimeric files in bins
chunked_nfl_files -- chunked non-chimeric files
out_pickle -- output pickle for saving PolishChunkTask objects
"""
n_bins = len(flnc_files)
n_nfl_chunks = max(1, len(chunked_nfl_files))
log.info("Writing %s ice_partial chunk tasks to %s.",
str(n_bins * n_nfl_chunks), out_pickle)
p = ChunkTasksPickle()
for i, flnc_file in enumerate(flnc_files):
log.debug("Processing cluster bin index=%s.", i)
cluster_out_dir = _get_cluster_out_dir(flnc_file)
for j, nfl_file in enumerate(chunked_nfl_files):
# Create Partial chunk tasks.
task_ = PartialChunkTask(cluster_bin_index=i,
flnc_file=flnc_file,
cluster_out_dir=cluster_out_dir,
nfl_file=nfl_file,
nfl_index=j,
n_nfl_chunks=n_nfl_chunks)
p.append(task_)
p.write(out_pickle)
log.info("Saved %s partial chunk tasks to %s.", str(n_bins * n_nfl_chunks),
out_pickle)
def run_main(cluster_chunks_pickle_file, ccs_file, output_json_file, max_nchunks):
"""Scatter items in cluster_chunks_pickle
Parameters:
cluster_chunks_pickle_file -- ChunkTasksPickle of ClusterChunkTask objects.
ccs_file -- ccs.consensusreadset.xml
output_json_file -- chunk.json
max_nchunks -- maximum # of chunks
"""
p = ChunkTasksPickle.read(cluster_chunks_pickle_file)
assert all([isinstance(r, ClusterChunkTask) for r in p])
out_dir = op.dirname(output_json_file)
# sort and group tasks
groups = p.sort_and_group_tasks(max_nchunks=max_nchunks)
# Writing chunk.json
base_name = "spawned_cluster_chunk"
chunks = []
spawned_pickles = []
for group_index in range(0, len(groups)):
chunk_id = "_".join([base_name, 'group', str(group_index)])
spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
d = {Constants.CHUNK_KEYS[0]: spawned_pickle_file,
Constants.CHUNK_KEYS[1]: ccs_file}
c = PipelineChunk(chunk_id, **d)
chunks.append(c)
spawned_pickles.append(spawned_pickle_file)
log.info("Spawning %s into %d files", cluster_chunks_pickle_file, len(groups))
p.spawn_pickles_by_groups(groups, spawned_pickles)
log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
# n_chunks = len(p)
# for i in range(0, n_chunks):
# chunk_id = "_".join([base_name, str(i)])
# spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
# d = {Constants.CHUNK_KEYS[0]: spawned_pickle_file,
# Constants.CHUNK_KEYS[1]: ccs_file}
# c = PipelineChunk(chunk_id, **d)
# chunks.append(c)
# spawned_pickles.append(spawned_pickle_file)
#
# log.info("Spawning %s into %s files", cluster_chunks_pickle_file, str(n_chunks))
# p.spawn_pickles(spawned_pickles)
# log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
log.info("Writing chunk.json to %s", output_json_file)
write_pipeline_chunks(chunks, output_json_file,
"created by %s" % Constants.TOOL_ID)
return 0
def resolved_tool_contract_runner(rtc):
"""Given resolved tool contract, run"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
p.sorted_by_attr(attr='cluster_bin_index')
assert all([isinstance(task, PartialChunkTask) for task in p])
with open(rtc.task.output_files[0], 'w') as writer:
for i, group in groupby(p, lambda x: x.cluster_bin_index):
gs = [g for g in group]
nfl_pickles_of_bin_i = [g.nfl_pickle for g in gs]
out_pickle = IceFiles(prog_name="", root_dir=gs[0].cluster_out_dir,
no_log_f=True).nfl_all_pickle_fn
log.info("Combining nfl pickles of cluster bin %s.", str(i))
log.debug("nfl pickles are: %s.", (", ".join(nfl_pickles_of_bin_i)))
log.debug("Output merged nfl pickle is %s.", out_pickle)
combine_nfl_pickles(splitted_pickles=nfl_pickles_of_bin_i, out_pickle=out_pickle)
writer.write("Merge nfl pickles of cluster bin %s DONE: %s\n" %
(i, out_pickle))
def run_main(partial_chunks_pickle_file, sentinel_file, ccs_file,
output_json_file, max_nchunks):
"""
Spawn partial Chunk Tasks in pickle.
Parameters:
partial_chunks_pickle_file -- ChunkTasksPickle of PartialChunkTask objects
ccs_file -- ccs dataset
sentinel_file -- sentinel file to connect pbsmrtpipe tasks
output_json -- chunk.json
"""
p = ChunkTasksPickle.read(partial_chunks_pickle_file)
assert all([isinstance(r, PartialChunkTask) for r in p])
out_dir = op.dirname(output_json_file)
# sort and group tasks
groups = p.sort_and_group_tasks(max_nchunks=max_nchunks)
# Writing chunk.json
base_name = "spawned_partial_chunk"
chunks = []
spawned_pickles = []
for group_index in range(0, len(groups)):
chunk_id = "_".join([base_name, 'group', str(group_index)])
spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
d = {
Constants.CHUNK_KEYS[0]: spawned_pickle_file,
Constants.CHUNK_KEYS[1]: sentinel_file,
Constants.CHUNK_KEYS[2]: ccs_file
}
c = PipelineChunk(chunk_id, **d)
chunks.append(c)
spawned_pickles.append(spawned_pickle_file)
log.info("Spawning %s into %d files", partial_chunks_pickle_file,
len(groups))
p.spawn_pickles_by_groups(groups=groups, out_pickle_fns=spawned_pickles)
log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
log.info("Writing chunk.json to %s", output_json_file)
write_pipeline_chunks(chunks, output_json_file,
"created by %s" % Constants.TOOL_ID)
return 0
def resolved_tool_contract_runner(rtc):
"""Given resolved tool contract, run"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
p.sorted_by_attr(attr='cluster_bin_index')
assert all([isinstance(task, PartialChunkTask) for task in p])
with open(rtc.task.output_files[0], 'w') as writer:
for i, group in groupby(p, lambda x: x.cluster_bin_index):
gs = [g for g in group]
nfl_pickles_of_bin_i = [g.nfl_pickle for g in gs]
out_pickle = IceFiles(prog_name="",
root_dir=gs[0].cluster_out_dir,
no_log_f=True).nfl_all_pickle_fn
log.info("Combining nfl pickles of cluster bin %s.", str(i))
log.debug("nfl pickles are: %s.",
(", ".join(nfl_pickles_of_bin_i)))
log.debug("Output merged nfl pickle is %s.", out_pickle)
combine_nfl_pickles(splitted_pickles=nfl_pickles_of_bin_i,
out_pickle=out_pickle)
writer.write("Merge nfl pickles of cluster bin %s DONE: %s\n" %
(i, out_pickle))
def run_main(polish_chunks_pickle_file, sentinel_file,
subreads_file, output_json_file, max_nchunks):
"""
Spawn polish Chunk Tasks in pickle.
Parameters:
polish_chunks_pickle_file -- ChunkTasksPickle of PolishChunkTask objects
subreads_file -- ccs dataset
sentinel_file -- sentinel file to connect pbsmrtpipe tasks.
output_json -- chunk.json
"""
p = ChunkTasksPickle.read(polish_chunks_pickle_file)
assert all([isinstance(r, PolishChunkTask) for r in p])
out_dir = op.dirname(output_json_file)
# sort and group tasks
groups = p.sort_and_group_tasks(max_nchunks=max_nchunks)
# Writing chunk.json
base_name = "spawned_polish_chunk"
chunks = []
spawned_pickles = []
for group_index in range(0, len(groups)):
chunk_id = "_".join([base_name, 'group', str(group_index)])
spawned_pickle_file = op.join(out_dir, chunk_id + ".pickle")
d = {Constants.CHUNK_KEYS[0]: spawned_pickle_file,
Constants.CHUNK_KEYS[1]: sentinel_file,
Constants.CHUNK_KEYS[2]: subreads_file}
c = PipelineChunk(chunk_id, **d)
chunks.append(c)
spawned_pickles.append(spawned_pickle_file)
log.info("Spawning %s into %d files", polish_chunks_pickle_file, len(groups))
p.spawn_pickles_by_groups(groups=groups, out_pickle_fns=spawned_pickles)
log.debug("Spawned files: %s.", ", ".join(spawned_pickles))
log.info("Writing chunk.json to %s", output_json_file)
write_pipeline_chunks(chunks, output_json_file,
"created by %s" % Constants.TOOL_ID)
return 0
def resolved_tool_contract_runner(rtc):
"""resolved tool contract runner."""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, PolishChunkTask) for task in p])
dummy_sentinel_file = rtc.task.input_files[1]
subread_set = rtc.task.input_files[2]
nproc = rtc.task.nproc
tmp_dir = rtc.task.tmpdir_resources[0].path \
if len(rtc.task.tmpdir_resources) > 0 else None
with open(rtc.task.output_files[0], 'w') as writer:
for task in p:
log.info("Running ice_polish on cluster bin %s, polish chunk %s/%s",
str(task.cluster_bin_index),
str(task.polish_index), str(task.n_polish_chunks))
log.debug("ice_quiver root_dir is %s", task.cluster_out_dir)
log.debug("consensus_isoforms is %s", task.consensus_isoforms_file)
task_runner(task=task, subread_set=subread_set, nproc=nproc, tmp_dir=tmp_dir)
writer.write("ice_polish of cluster bin %s, polish chunk %s/%s in %s is DONE.\n" %
(task.cluster_bin_index, task.polish_index, task.n_polish_chunks,
task.cluster_out_dir))
def resolved_tool_contract_runner(rtc):
"""
For each cluster bin, create summary.json, cluster_report.csv,
hq_isoforms.fa|fq, lq_isoforms.fa|fq
"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, PolishChunkTask) for task in p])
p.sorted_by_attr(attr='cluster_bin_index')
opts = rtc.task.options
ipq_opts = IceQuiverHQLQOptions(qv_trim_5=opts[Constants.QV_TRIM_FIVEPRIME_ID],
qv_trim_3=opts[Constants.QV_TRIM_THREEPRIME_ID],
hq_quiver_min_accuracy=opts[Constants.HQ_QUIVER_MIN_ACCURACY_ID])
with open(rtc.task.output_files[0], 'w') as writer:
for cluster_bin_index, cluster_out_dir in p.sorted_no_redundant_cluster_bins():
log.info("ice_quiver_postprocess of cluster bin index %s in %s.",
str(cluster_bin_index), str(cluster_out_dir))
good_hq, bad_hq = \
ice_quiver_postprocess_a_cluster_bin(cluster_out_dir=cluster_out_dir,
ipq_opts=ipq_opts)
writer.write("ice_quiver_postprocess of cluster bin index %s in %s DONE:\n%s\n%s\n" %
(cluster_bin_index, cluster_out_dir, good_hq, bad_hq))
def resolved_tool_contract_runner(rtc):
"""run all tasks in cluster_chunks.pickle given rtc"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
ccs_file = rtc.task.input_files[1]
assert op.exists(ccs_file)
nproc = rtc.task.nproc
use_finer_qv = False
#if rtc.task.options.get(Constants.USE_FINER_QV_ID, False):
# use_finer_qv = True
with open(rtc.task.output_files[0], 'w') as writer:
for i, task in enumerate(p):
args = task_to_args(task=task, ccs_file=ccs_file,
nproc=nproc, use_finer_qv=use_finer_qv)
log.info("ARGUMENTS of Task %s/%s:\n%s", str(i), str(len(p)), str(args))
log.info("Running ICE on cluster bin %s", task.cluster_bin_index)
PBTranscript(args, subCommand="cluster").start()
writer.write("ICE of cluster bin %s in %s is DONE: %s\n" %
(task.cluster_bin_index, task.cluster_out_dir,
task.consensus_isoforms_file))
def create_cluster_pickle(flnc_files, out_pickle):
"""Create cluster chunk task pickle.
Parameters:
n_bins -- number of bins
flnc_files -- full-length non-chimeric files in bins
out_pickle -- output pickle for saving ClusterChunkTask objects
"""
n_bins = len(flnc_files)
log.info("Writing %s cluster chunk tasks to %s.", str(n_bins), out_pickle)
p = ChunkTasksPickle()
for i, flnc_file in enumerate(flnc_files):
log.debug("Processing cluster bin index=%s.", i)
cluster_out_dir = _get_cluster_out_dir(flnc_file)
# Create Cluster chunk tasks.
task_ = ClusterChunkTask(cluster_bin_index=i, flnc_file=flnc_file,
cluster_out_dir=cluster_out_dir)
p.append(task_)
p.write(out_pickle)
log.info("Saved %s cluster chunk tasks to %s.", str(n_bins), out_pickle)
def resolved_tool_contract_runner(rtc):
"""
For each cluster bin, create summary.json, cluster_report.csv,
hq_isoforms.fa|fq, lq_isoforms.fa|fq
Finally, merge all cluster bins and save all outputs to 'combined'.
"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
p.sorted_by_attr(attr='cluster_bin_index')
opts = rtc.task.options
ipq_opts = IceQuiverHQLQOptions(
qv_trim_5=opts[Constants.QV_TRIM_FIVEPRIME_ID],
qv_trim_3=opts[Constants.QV_TRIM_THREEPRIME_ID],
hq_quiver_min_accuracy=opts[Constants.HQ_QUIVER_MIN_ACCURACY_ID])
sample_name = get_sample_name(
input_sample_name=opts[Constants.SAMPLE_NAME_ID])
out_consensus_isoforms_cs = rtc.task.output_files[0]
out_summary = rtc.task.output_files[1]
out_report = rtc.task.output_files[2]
out_hq_cs = rtc.task.output_files[3]
out_hq_fq = rtc.task.output_files[4]
out_lq_cs = rtc.task.output_files[5]
out_lq_fq = rtc.task.output_files[6]
out_hq_lq_prefix_dict_pickle = rtc.task.output_files[7]
assert out_consensus_isoforms_cs.endswith(".contigset.xml")
assert out_hq_cs.endswith(".contigset.xml")
assert out_lq_cs.endswith(".contigset.xml")
out_consensus_isoforms_fa = out_consensus_isoforms_cs.replace(
".contigset.xml", ".fasta")
out_hq_fa = out_hq_cs.replace('.contigset.xml', '.fasta')
out_lq_fa = out_lq_cs.replace('.contigset.xml', '.fasta')
hq_fq_fns, lq_fq_fns = [], []
split_uc_pickles, split_partial_uc_pickles = [], []
split_consensus_isoforms = []
cluster_bin_indices = [task.cluster_bin_index for task in p]
cluster_out_dirs = [task.cluster_out_dir for task in p]
# sanity check that Cluster indices are unique!
assert len(set(cluster_bin_indices)) == len(cluster_bin_indices)
for task in p:
ice_pq = IceQuiverPostprocess(root_dir=task.cluster_out_dir,
ipq_opts=ipq_opts)
hq_fq_fns.append(ice_pq.quivered_good_fq)
lq_fq_fns.append(ice_pq.quivered_bad_fq)
split_uc_pickles.append(ice_pq.final_pickle_fn)
split_partial_uc_pickles.append(ice_pq.nfl_all_pickle_fn)
split_consensus_isoforms.append(ice_pq.final_consensus_fa)
combined_dir = op.join(op.dirname(op.dirname(cluster_out_dirs[0])),
"combined")
mkdir(combined_dir)
combined_files = CombinedFiles(combined_dir)
log.info("Combining results of all cluster bins to %s.", combined_dir)
log.info("Merging HQ|LQ isoforms from all cluster bins.")
log.info("HQ isoforms are: %s.", ",".join(hq_fq_fns))
log.info("LQ isoforms are: %s.", ",".join(lq_fq_fns))
combine_polished_isoforms(
split_indices=cluster_bin_indices,
split_hq_fns=hq_fq_fns,
split_lq_fns=lq_fq_fns,
combined_hq_fa=combined_files.all_hq_fa,
combined_hq_fq=combined_files.all_hq_fq,
combined_lq_fa=combined_files.all_lq_fa,
combined_lq_fq=combined_files.all_lq_fq,
hq_lq_prefix_dict_pickle=combined_files.hq_lq_prefix_dict_pickle,
sample_name=sample_name)
ln(combined_files.all_hq_fa, out_hq_fa) #'HQ isoforms'
ln(combined_files.all_hq_fq, out_hq_fq) #'HQ isoforms'
ln(combined_files.all_lq_fa, out_lq_fa) #'LQ isoforms'
ln(combined_files.all_lq_fq, out_lq_fq) #'LQ isoforms'
ln(combined_files.hq_lq_prefix_dict_pickle, out_hq_lq_prefix_dict_pickle)
as_contigset(out_hq_fa, out_hq_cs)
as_contigset(out_lq_fa, out_lq_cs)
log.info("Merging consensus isoforms from all cluster bins.")
combine_consensus_isoforms(split_indices=cluster_bin_indices,
split_files=split_consensus_isoforms,
combined_consensus_isoforms_fa=combined_files.
all_consensus_isoforms_fa,
sample_name=sample_name)
ln(combined_files.all_consensus_isoforms_fa, out_consensus_isoforms_fa)
#consensus isoforms
as_contigset(out_consensus_isoforms_fa, out_consensus_isoforms_cs)
log.info("Writing cluster summary to %s",
combined_files.all_cluster_summary_fn)
write_cluster_summary(summary_fn=combined_files.all_cluster_summary_fn,
isoforms_fa=out_consensus_isoforms_cs,
hq_fa=out_hq_fa,
lq_fa=out_lq_fa)
ln(combined_files.all_cluster_summary_fn, out_summary) # "cluster summary"
log.info("Writing cluster report to %s",
combined_files.all_cluster_report_fn)
write_combined_cluster_report(
split_indices=cluster_bin_indices,
split_uc_pickles=split_uc_pickles,
split_partial_uc_pickles=split_partial_uc_pickles,
report_fn=combined_files.all_cluster_report_fn,
sample_name=sample_name)
ln(combined_files.all_cluster_report_fn, out_report) # "cluster report"
def resolved_tool_contract_runner(rtc):
"""
For each cluster bin, create summary.json, cluster_report.csv,
hq_isoforms.fa|fq, lq_isoforms.fa|fq
Finally, merge all cluster bins and save all outputs to 'combined'.
"""
p = ChunkTasksPickle.read(rtc.task.input_files[0])
assert all([isinstance(task, ClusterChunkTask) for task in p])
p.sorted_by_attr(attr='cluster_bin_index')
opts = rtc.task.options
ipq_opts = IceQuiverHQLQOptions(qv_trim_5=opts[Constants.QV_TRIM_FIVEPRIME_ID],
qv_trim_3=opts[Constants.QV_TRIM_THREEPRIME_ID],
hq_quiver_min_accuracy=opts[Constants.HQ_QUIVER_MIN_ACCURACY_ID])
sample_name = get_sample_name(input_sample_name=opts[Constants.SAMPLE_NAME_ID])
out_consensus_isoforms_cs = rtc.task.output_files[0]
out_summary = rtc.task.output_files[1]
out_report = rtc.task.output_files[2]
out_hq_cs = rtc.task.output_files[3]
out_hq_fq = rtc.task.output_files[4]
out_lq_cs = rtc.task.output_files[5]
out_lq_fq = rtc.task.output_files[6]
out_hq_lq_prefix_dict_pickle = rtc.task.output_files[7]
assert out_consensus_isoforms_cs.endswith(".contigset.xml")
assert out_hq_cs.endswith(".contigset.xml")
assert out_lq_cs.endswith(".contigset.xml")
out_consensus_isoforms_fa = out_consensus_isoforms_cs.replace(".contigset.xml", ".fasta")
out_hq_fa = out_hq_cs.replace('.contigset.xml', '.fasta')
out_lq_fa = out_lq_cs.replace('.contigset.xml', '.fasta')
hq_fq_fns, lq_fq_fns = [], []
split_uc_pickles, split_partial_uc_pickles = [], []
split_consensus_isoforms = []
cluster_bin_indices = [task.cluster_bin_index for task in p]
cluster_out_dirs = [task.cluster_out_dir for task in p]
# sanity check that Cluster indices are unique!
assert len(set(cluster_bin_indices)) == len(cluster_bin_indices)
for task in p:
ice_pq = IceQuiverPostprocess(root_dir=task.cluster_out_dir,
ipq_opts=ipq_opts)
hq_fq_fns.append(ice_pq.quivered_good_fq)
lq_fq_fns.append(ice_pq.quivered_bad_fq)
split_uc_pickles.append(ice_pq.final_pickle_fn)
split_partial_uc_pickles.append(ice_pq.nfl_all_pickle_fn)
split_consensus_isoforms.append(ice_pq.final_consensus_fa)
combined_dir = op.join(op.dirname(op.dirname(cluster_out_dirs[0])), "combined")
mkdir(combined_dir)
combined_files = CombinedFiles(combined_dir)
log.info("Combining results of all cluster bins to %s.", combined_dir)
log.info("Merging HQ|LQ isoforms from all cluster bins.")
log.info("HQ isoforms are: %s.", ",".join(hq_fq_fns))
log.info("LQ isoforms are: %s.", ",".join(lq_fq_fns))
combine_polished_isoforms(split_indices=cluster_bin_indices,
split_hq_fns=hq_fq_fns,
split_lq_fns=lq_fq_fns,
combined_hq_fa=combined_files.all_hq_fa,
combined_hq_fq=combined_files.all_hq_fq,
combined_lq_fa=combined_files.all_lq_fa,
combined_lq_fq=combined_files.all_lq_fq,
hq_lq_prefix_dict_pickle=combined_files.hq_lq_prefix_dict_pickle,
sample_name=sample_name)
ln(combined_files.all_hq_fa, out_hq_fa) #'HQ isoforms'
ln(combined_files.all_hq_fq, out_hq_fq) #'HQ isoforms'
ln(combined_files.all_lq_fa, out_lq_fa) #'LQ isoforms'
ln(combined_files.all_lq_fq, out_lq_fq) #'LQ isoforms'
ln(combined_files.hq_lq_prefix_dict_pickle, out_hq_lq_prefix_dict_pickle)
as_contigset(out_hq_fa, out_hq_cs)
as_contigset(out_lq_fa, out_lq_cs)
log.info("Merging consensus isoforms from all cluster bins.")
combine_consensus_isoforms(split_indices=cluster_bin_indices,
split_files=split_consensus_isoforms,
combined_consensus_isoforms_fa=combined_files.all_consensus_isoforms_fa,
sample_name=sample_name)
ln(combined_files.all_consensus_isoforms_fa, out_consensus_isoforms_fa)
#consensus isoforms
as_contigset(out_consensus_isoforms_fa, out_consensus_isoforms_cs)
log.info("Writing cluster summary to %s", combined_files.all_cluster_summary_fn)
write_cluster_summary(summary_fn=combined_files.all_cluster_summary_fn,
isoforms_fa=out_consensus_isoforms_cs,
hq_fa=out_hq_fa,
lq_fa=out_lq_fa)
ln(combined_files.all_cluster_summary_fn, out_summary) # "cluster summary"
log.info("Writing cluster report to %s", combined_files.all_cluster_report_fn)
write_combined_cluster_report(split_indices=cluster_bin_indices,
split_uc_pickles=split_uc_pickles,
split_partial_uc_pickles=split_partial_uc_pickles,
report_fn=combined_files.all_cluster_report_fn,
sample_name=sample_name)
ln(combined_files.all_cluster_report_fn, out_report) # "cluster report"