def run_rmarkdown(start=0, sample= "", theme= "",
workdir=None, outdir=None, timeout=TIMEOUT,nthreads=1, kobas=True):
logger.info("Get rmarkdown report for %s"%sample)
report_log = os.path.join(workdir, "report.log")
report_log_fd = open(report_log, "w")
step=0
msg = "Get report for %s"%sample
sma3s_summary = "%s/%s-sma3s-summary.tsv" %(os.path.join(outdir,"annotation"), sample)
sma3s_table = "%s/%s-sma3s-table.tsv" %(os.path.join(outdir,"annotation"), sample)
sma3s_go = "%s/%s-go.tsv" %(os.path.join(outdir,"annotation"), sample)
sma3s_ko = "%s/%s-ko.tsv" %(os.path.join(outdir,"annotation"), sample)
engine=""
if os.path.exists(sma3s_go and sma3s_ko):
engine="0"
elif os.path.exists(sma3s_summary and sma3s_table):
engine="1"
else:
logger.error("The files of annotation did not existed!")
os._exit(-1)
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if kobas: #convert to str
kobas="1"
else:
kobas="0"
command='''\
python /opt/Auxtools/rmd_creator.py -i %(in)s -w %(work)s \
-t /opt/Auxtools/rmarkdown/template.Rmd \
-s %(sam)s -o %(in)s/%(sam)s.Rmd -m %(theme)s -k %(kobas)s -e %(engine)s && \
R -e 'rmarkdown::render(\\"%(in)s/%(sam)s.Rmd\\")' && \
rm %(in)s/%(sam)s.Rmd \
''' % {
'in': outdir,
'work': workdir,
'sam': sample,
'theme': theme,
'kobas': kobas,
'engine': engine
}
logger.info(command)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=report_log_fd, cmd_log=report_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
def run_fusioncatcher(data_dir="",
input="",
start=0,
fusioncatcher=FUSIONCATCHER,
fusioncatcher_opts="",
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running RNA fusion detection (FusionCatcher) for %s" % sample)
if not os.path.exists(data_dir):
logger.error("Aborting!")
raise Exception("No data directory %s" % data_dir)
work_fusioncatcher = os.path.join(workdir, "fusioncatcher", sample)
create_dirs([work_fusioncatcher])
fusioncatcher_log = os.path.join(work_fusioncatcher, "fusioncatcher.log")
fusioncatcher_log_fd = open(fusioncatcher_log, "w")
if nthreads > 1:
if "-p " not in fusioncatcher_opts:
fusioncatcher_opts += " -p %d" % nthreads
msg = "Run FusionCatcher for %s" % sample
command = "%s -d %s -i %s --start %d -o %s" % (
fusioncatcher, data_dir, input, start, work_fusioncatcher)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=fusioncatcher_log_fd,
cmd_log=fusioncatcher_log_fd,
msg=msg,
timeout=timeout)
out_fusioncatcher = os.path.join(outdir, "fusioncatcher", sample)
create_dirs([out_fusioncatcher])
msg = "Copy predictions to output directory for %s." % sample
if os.path.exists("%s/final-list_candidate-fusion-genes.txt" %
work_fusioncatcher):
command = "cp %s/final-list_candidate-fusion-genes.txt %s/final-list_candidate-fusion-genes.txt" % (
work_fusioncatcher, out_fusioncatcher)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=fusioncatcher_log_fd,
cmd_log=fusioncatcher_log,
msg=msg,
timeout=timeout)
fusions = ""
if os.path.exists("%s/final-list_candidate-fusion-genes.txt" %
out_fusioncatcher):
logger.info("FusionCatcher was successfull!")
logger.info(
"Output fusions: %s/final-list_candidate-fusion-genes.txt" %
out_fusioncatcher)
fusions = "%s/final-list_candidate-fusion-genes.txt" % out_fusioncatcher
else:
logger.info("FusionCatcher failed!")
return fusions
def run_age_single(intervals_bed=None, region_list=[], contig_dict={}, reference=None, assembly=None, pad=AGE_PAD,
age=None, truncation_pad_read_age = AGE_TRUNCATION_PAD,
max_interval_len_truncation_age = AGE_MAX_INTERVAL_TRUNCATION,
dist_to_expected_bp = AGE_DIST_TO_BP, min_del_subalign_len = MIN_DEL_SUBALIGN_LENGTH,
min_inv_subalign_len = MIN_INV_SUBALIGN_LENGTH, age_window = AGE_WINDOW_SIZE,
age_workdir=None, timeout=AGE_TIMEOUT, keep_temp=False, myid=0):
thread_logger = logging.getLogger("%s-%s" % (run_age_single.__name__, multiprocessing.current_process()))
bedtools_intervals = []
intervals_bedtool = pybedtools.BedTool(intervals_bed)
assembly_fasta = pysam.Fastafile(assembly) if assembly else None
reference_fasta = pysam.Fastafile(reference)
breakpoints_bed = None
thread_logger.info("Will process %d intervals" % (len(region_list)))
try:
for region in region_list:
bedtools_interval = pybedtools.Interval(region[0], region[1], region[3])
matching_intervals = [interval for interval in intervals_bedtool if (
interval.start == bedtools_interval.start and interval.end == bedtools_interval.end and interval.chrom == bedtools_interval.chrom)]
if not matching_intervals:
thread_logger.info("Matching interval not found for %s" % (str(bedtools_interval)))
matching_interval = bedtools_interval
else:
matching_interval = matching_intervals[0]
thread_logger.info("Matching interval %s" % (str(matching_interval)))
sc_locations = []
try:
sc_locations = map(int, json.loads(base64.b64decode(matching_interval.name.split(",")[0]))["SC_LOCATIONS"].split(","))
except:
pass
if region not in contig_dict:
continue
if not contig_dict[region]:
continue
region_object = SVRegion(region[0], region[1], region[2], region[3])
if region_object.pos1 - pad < 0:
thread_logger.error("Region too close to start of chromosome. Skipping.")
continue
reference_sequence = reference_fasta.fetch(reference=region_object.chrom1, start=region_object.pos1 - pad,
end=region_object.pos2 + pad)
region_name = "%s.%d.%d" % (region_object.chrom1, region_object.pos1, region_object.pos2)
ref_name = os.path.join(age_workdir, "%s.ref.fa" % region_name)
thread_logger.info("Writing the ref sequence for region %s" % region_name)
with open(ref_name, "w") as file_handle:
file_handle.write(">{}.ref\n{}".format(region_name, reference_sequence))
age_records = []
thread_logger.info("Processing %d contigs for region %s" % (len(contig_dict[region]), str(region_object)))
for contig in contig_dict[region]:
thread_logger.info(
"Writing the assembeled sequence %s of length %s" % (contig.raw_name, contig.sequence_len))
tr_region=[]
if region_object.length()>max_interval_len_truncation_age and contig.sv_type in ["INV","DEL","DUP"]:
# For large SVs, middle sequences has no effect on genotyping. So, we truncate middle region of reference to speed up
thread_logger.info("Truncate the reference sequence.")
truncate_start = pad + dist_to_expected_bp + truncation_pad_read_age +1
truncate_end = len(reference_sequence) - (pad + dist_to_expected_bp + truncation_pad_read_age)
reference_sequence_tr=reference_sequence[0:truncate_start-1]+reference_sequence[truncate_end:]
region_name_tr = "%s.%d.%d.tr_%d_%d" % (region_object.chrom1, region_object.pos1, region_object.pos2,truncate_start,truncate_end)
ref_name_tr = os.path.join(age_workdir, "%s.ref.fa" % region_name_tr)
thread_logger.info("Writing the truncated ref sequence for region %s, contig %s" % (region_name_tr, contig.raw_name))
with open(ref_name_tr, "w") as file_handle:
file_handle.write(">{}.ref\n{}".format(region_name_tr, reference_sequence_tr))
ref_len = len(reference_sequence_tr)
ref_f_name = ref_name_tr
tr_region = [truncate_start,truncate_end-truncate_start+1]
else:
ref_len = region_object.length()
ref_f_name = ref_name
if contig.sequence_len * ref_len >= 100000000:
thread_logger.info("Skipping contig because AGE problem is large (contig_len = %d , ref_len= %d)"%(contig.sequence_len, ref_len))
continue
contig_sequence = assembly_fasta.fetch(contig.raw_name)
prefix = get_age_file_prefix(contig)
asm_name = os.path.join(age_workdir, "%s.as.fa" % prefix)
out = os.path.join(age_workdir, "%s.age.out" % prefix)
err = os.path.join(age_workdir, "%s.age.err" % prefix)
fd_out = open(out, "w")
fd_err = open(err, "w")
with open(asm_name, "w") as file_handle:
file_handle.write(">{}.as\n{}".format(region_name, contig_sequence))
age_cmd = "%s %s -both -go=-6 %s %s" % (
age, "-inv" if contig.sv_type == "INV" else "-tdup" if contig.sv_type == "DUP" else "-indel",
ref_f_name, asm_name)
cmd_runner = TimedExternalCmd(age_cmd, thread_logger)
retcode = cmd_runner.run(timeout=timeout, cmd_log_fd_out=fd_out, cmd_log_fd_err=fd_err)
fd_out.close()
fd_err.close()
if retcode == 0:
age_record = AgeRecord(out,tr_region_1=tr_region)
if len(age_record.inputs) == 2:
age_record.contig = contig
age_record.set_assembly_contig(contig_sequence)
age_records.append(age_record)
else:
thread_logger.error("Number of inputs != 2 in age output file %s. Skipping." % out)
if not keep_temp:
os.remove(asm_name)
os.remove(err)
if tr_region:
os.remove(ref_name_tr)
unique_age_records = get_unique_age_records(age_records)
thread_logger.info("Unique %d AGE records for region %s" % (len(unique_age_records), str(region_object)))
for age_record in unique_age_records:
thread_logger.info(str(age_record))
sv_types = list(set([age_record.contig.sv_type for age_record in unique_age_records]))
if len(sv_types) != 1:
thread_logger.error("Some problem. Mixed SV types for this interval %s" % (str(sv_types)))
else:
sv_type = sv_types[0]
thread_logger.info("Processing region of type %s" % sv_type)
breakpoints, info_dict = process_age_records(unique_age_records, sv_type=sv_type,
pad=pad, dist_to_expected_bp=dist_to_expected_bp,
min_del_subalign_len=min_del_subalign_len,
min_inv_subalign_len=min_inv_subalign_len,
age_window=age_window, sc_locations=sc_locations)
bedtools_fields = matching_interval.fields
if len(breakpoints) == 1 and sv_type == "INS":
bedtools_fields += map(str, [breakpoints[0][0], breakpoints[0][0] + 1, breakpoints[0][1], breakpoints[0][2]])
elif len(breakpoints) == 2 and (sv_type in ["DEL","INV","DUP"]):
bedtools_fields += map(str, breakpoints + [breakpoints[1] - breakpoints[0]] + ["."])
else:
bedtools_fields += map(str, [bedtools_fields[1], bedtools_fields[2], -1, "."])
bedtools_fields[3] += ";AS"
bedtools_fields.append(base64.b64encode(json.dumps(info_dict)))
thread_logger.info("Writing out fields %s" % (str(bedtools_fields)))
bedtools_intervals.append(pybedtools.create_interval_from_list(bedtools_fields))
if not keep_temp:
os.remove(ref_name)
except Exception as e:
thread_logger.error('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
if assembly_fasta:
assembly_fasta.close()
reference_fasta.close()
thread_logger.info("Writing %d intervals" % (len(bedtools_intervals)))
if bedtools_intervals:
breakpoints_bed = os.path.join(age_workdir, "%d_breakpoints.bed" % myid)
pybedtools.BedTool(bedtools_intervals).saveas(breakpoints_bed)
return breakpoints_bed
def run_spades_single(intervals=[],
bams=[],
spades=None,
spades_options="",
work=None,
pad=SPADES_PAD,
timeout=SPADES_TIMEOUT,
isize_min=ISIZE_MIN,
isize_max=ISIZE_MAX,
stop_on_fail=False,
max_read_pairs=EXTRACTION_MAX_READ_PAIRS):
thread_logger = logging.getLogger(
"%s-%s" %
(run_spades_single.__name__, multiprocessing.current_process()))
if not os.path.isdir(work):
thread_logger.info("Creating %s" % work)
os.makedirs(work)
merged_contigs = open(os.path.join(work, "merged.fa"), "w")
spades_log_fd = open(os.path.join(work, "spades.log"), "w")
extract_fns = [extract_pairs.all_pair_hq, extract_pairs.non_perfect_hq]
try:
bam_handles = [pysam.Samfile(bam, "rb") for bam in bams]
for interval in intervals:
region = "%s:%d-%d" % (str(
interval.chrom), interval.start, interval.end)
thread_logger.info("Processing interval %s" %
(str(interval).strip()))
sv_type = interval.name.split(",")[1]
extraction_counts = extract_pairs.extract_read_pairs(
bam_handles,
region,
"%s/" % work,
extract_fns,
pad=pad,
max_read_pairs=max_read_pairs,
sv_type=sv_type)
all_pair_count = extraction_counts[0][1]
for fn_id, ((end1, end2),
extracted_count) in enumerate(extraction_counts):
extract_fn_name = extract_fns[fn_id].__name__
if fn_id > 0 and extracted_count == all_pair_count:
thread_logger.info(
"Skipping assembly from %s since read count same as all_pairs"
% extract_fn_name)
continue
if extracted_count >= 5:
extra_opt = "--sc" if not fn_id == 0 else ""
spades_log_fd.write(
"Running spades for interval %s with extraction function %s\n"
% (str(interval).strip(), extract_fn_name))
cmd = TimedExternalCmd(
"%s -1 %s -2 %s -o %s/spades_%s/ -m 4 -t 1 --phred-offset 33 %s %s"
% (spades, end1, end2, work, extract_fn_name,
extra_opt, spades_options), thread_logger)
retcode = cmd.run(cmd_log_fd_out=spades_log_fd,
timeout=timeout)
if retcode == 0:
append_contigs(
os.path.join(work, "spades_%s/contigs.fasta") %
extract_fn_name, interval, merged_contigs, fn_id,
sv_type)
elif not cmd.did_timeout:
thread_logger.error("Spades failed")
if stop_on_fail:
thread_logger.error("Aborting!")
raise Exception(
"Spades failure on interval %s for extraction function %s\n"
% (str(interval).strip(), extract_fn_name))
else:
thread_logger.info(
"Too few read pairs (%d) extracted. Skipping assembly."
% extracted_count)
for bam_handle in bam_handles:
bam_handle.close()
except Exception as e:
thread_logger.error('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
merged_contigs.close()
return os.path.abspath(merged_contigs.name)
def run_spades_single(intervals=[], bams=[], spades=None, spades_options="", work=None, pad=SPADES_PAD, timeout=SPADES_TIMEOUT,
isize_min=ISIZE_MIN,
isize_max=ISIZE_MAX, stop_on_fail=False, max_read_pairs=EXTRACTION_MAX_READ_PAIRS):
thread_logger = logging.getLogger("%s-%s" % (run_spades_single.__name__, multiprocessing.current_process()))
if not os.path.isdir(work):
thread_logger.info("Creating %s" % work)
os.makedirs(work)
merged_contigs = open(os.path.join(work, "merged.fa"), "w")
spades_log_fd = open(os.path.join(work, "spades.log"), "w")
extract_fns = [extract_pairs.all_pair_hq, extract_pairs.non_perfect_hq]
try:
bam_handles = [pysam.Samfile(bam, "rb") for bam in bams]
for interval in intervals:
region = "%s:%d-%d" % (str(interval.chrom), interval.start, interval.end)
thread_logger.info("Processing interval %s" % (str(interval).strip()))
sv_type = interval.name.split(",")[1]
extraction_counts = extract_pairs.extract_read_pairs(bam_handles, region, "%s/" % work, extract_fns, pad=pad,
max_read_pairs=max_read_pairs, sv_type=sv_type)
all_pair_count = extraction_counts[0][1]
for fn_id, ((end1, end2), extracted_count) in enumerate(extraction_counts):
extract_fn_name = extract_fns[fn_id].__name__
if fn_id > 0 and extracted_count == all_pair_count:
thread_logger.info("Skipping assembly from %s since read count same as all_pairs" % extract_fn_name)
continue
if extracted_count >= 5:
extra_opt = "--sc" if not fn_id == 0 else ""
spades_log_fd.write("Running spades for interval %s with extraction function %s\n" % (
str(interval).strip(), extract_fn_name))
cmd = TimedExternalCmd("%s -1 %s -2 %s -o %s/spades_%s/ -m 4 -t 1 --phred-offset 33 %s %s" % (
spades, end1, end2, work, extract_fn_name, extra_opt, spades_options), thread_logger)
retcode = cmd.run(cmd_log_fd_out=spades_log_fd, timeout=timeout)
if retcode == 0:
append_contigs(os.path.join(work, "spades_%s/contigs.fasta") % extract_fn_name, interval,
merged_contigs, fn_id, sv_type)
elif not cmd.did_timeout:
thread_logger.error("Spades failed")
if stop_on_fail:
thread_logger.error("Aborting!")
raise Exception("Spades failure on interval %s for extraction function %s\n" % (
str(interval).strip(), extract_fn_name))
else:
thread_logger.info("Too few read pairs (%d) extracted. Skipping assembly." % extracted_count)
for bam_handle in bam_handles:
bam_handle.close()
except Exception as e:
thread_logger.error('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
merged_contigs.close()
return os.path.abspath(merged_contigs.name)
def run_age_single(intervals_bed=None,
region_list=[],
contig_dict={},
reference=None,
assembly=None,
pad=AGE_PAD,
age=None,
truncation_pad_read_age=AGE_TRUNCATION_PAD,
max_interval_len_truncation_age=AGE_MAX_INTERVAL_TRUNCATION,
dist_to_expected_bp=AGE_DIST_TO_BP,
min_del_subalign_len=MIN_DEL_SUBALIGN_LENGTH,
min_inv_subalign_len=MIN_INV_SUBALIGN_LENGTH,
age_window=AGE_WINDOW_SIZE,
age_workdir=None,
timeout=AGE_TIMEOUT,
keep_temp=False,
myid=0):
thread_logger = logging.getLogger(
"%s-%s" % (run_age_single.__name__, multiprocessing.current_process()))
bedtools_intervals = []
intervals_bedtool = pybedtools.BedTool(intervals_bed)
assembly_fasta = pysam.Fastafile(assembly) if assembly else None
reference_fasta = pysam.Fastafile(reference)
breakpoints_bed = None
thread_logger.info("Will process %d intervals" % (len(region_list)))
try:
for region in region_list:
bedtools_interval = pybedtools.Interval(region[0], region[1],
region[3])
matching_intervals = [
interval for interval in intervals_bedtool
if (interval.start == bedtools_interval.start
and interval.end == bedtools_interval.end
and interval.chrom == bedtools_interval.chrom)
]
if not matching_intervals:
thread_logger.info("Matching interval not found for %s" %
(str(bedtools_interval)))
matching_interval = bedtools_interval
else:
matching_interval = matching_intervals[0]
thread_logger.info("Matching interval %s" %
(str(matching_interval)))
sc_locations = []
try:
sc_locations = map(
int,
json.loads(
base64.b64decode(matching_interval.name.split(",")[0]))
["SC_LOCATIONS"].split(","))
except:
pass
if region not in contig_dict:
continue
if not contig_dict[region]:
continue
region_object = SVRegion(region[0], region[1], region[2],
region[3])
if region_object.pos1 - pad < 0:
thread_logger.error(
"Region too close to start of chromosome. Skipping.")
continue
reference_sequence = reference_fasta.fetch(
reference=region_object.chrom1,
start=region_object.pos1 - pad,
end=region_object.pos2 + pad)
region_name = "%s.%d.%d" % (region_object.chrom1,
region_object.pos1, region_object.pos2)
ref_name = os.path.join(age_workdir, "%s.ref.fa" % region_name)
thread_logger.info("Writing the ref sequence for region %s" %
region_name)
with open(ref_name, "w") as file_handle:
file_handle.write(">{}.ref\n{}".format(region_name,
reference_sequence))
age_records = []
thread_logger.info("Processing %d contigs for region %s" %
(len(contig_dict[region]), str(region_object)))
for contig in contig_dict[region]:
thread_logger.info(
"Writing the assembeled sequence %s of length %s" %
(contig.raw_name, contig.sequence_len))
tr_region = []
if region_object.length(
) > max_interval_len_truncation_age and contig.sv_type in [
"INV", "DEL", "DUP"
]:
# For large SVs, middle sequences has no effect on genotyping. So, we truncate middle region of reference to speed up
thread_logger.info("Truncate the reference sequence.")
truncate_start = pad + dist_to_expected_bp + truncation_pad_read_age + 1
truncate_end = len(reference_sequence) - (
pad + dist_to_expected_bp + truncation_pad_read_age)
reference_sequence_tr = reference_sequence[
0:truncate_start -
1] + reference_sequence[truncate_end:]
region_name_tr = "%s.%d.%d.tr_%d_%d" % (
region_object.chrom1, region_object.pos1,
region_object.pos2, truncate_start, truncate_end)
ref_name_tr = os.path.join(age_workdir,
"%s.ref.fa" % region_name_tr)
thread_logger.info(
"Writing the truncated ref sequence for region %s, contig %s"
% (region_name_tr, contig.raw_name))
with open(ref_name_tr, "w") as file_handle:
file_handle.write(">{}.ref\n{}".format(
region_name_tr, reference_sequence_tr))
ref_len = len(reference_sequence_tr)
ref_f_name = ref_name_tr
tr_region = [
truncate_start, truncate_end - truncate_start + 1
]
else:
ref_len = region_object.length()
ref_f_name = ref_name
if contig.sequence_len * ref_len >= 100000000:
thread_logger.info(
"Skipping contig because AGE problem is large (contig_len = %d , ref_len= %d)"
% (contig.sequence_len, ref_len))
continue
contig_sequence = assembly_fasta.fetch(contig.raw_name)
prefix = get_age_file_prefix(contig)
asm_name = os.path.join(age_workdir, "%s.as.fa" % prefix)
out = os.path.join(age_workdir, "%s.age.out" % prefix)
err = os.path.join(age_workdir, "%s.age.err" % prefix)
fd_out = open(out, "w")
fd_err = open(err, "w")
with open(asm_name, "w") as file_handle:
file_handle.write(">{}.as\n{}".format(
region_name, contig_sequence))
age_cmd = "%s %s -both -go=-6 %s %s" % (
age, "-inv" if contig.sv_type == "INV" else
"-tdup" if contig.sv_type == "DUP" else "-indel",
ref_f_name, asm_name)
cmd_runner = TimedExternalCmd(age_cmd, thread_logger)
retcode = cmd_runner.run(timeout=timeout,
cmd_log_fd_out=fd_out,
cmd_log_fd_err=fd_err)
fd_out.close()
fd_err.close()
if retcode == 0:
age_record = AgeRecord(out, tr_region_1=tr_region)
if len(age_record.inputs) == 2:
age_record.contig = contig
age_record.set_assembly_contig(contig_sequence)
age_records.append(age_record)
else:
thread_logger.error(
"Number of inputs != 2 in age output file %s. Skipping."
% out)
if not keep_temp:
os.remove(asm_name)
os.remove(err)
if tr_region:
os.remove(ref_name_tr)
unique_age_records = get_unique_age_records(age_records)
thread_logger.info("Unique %d AGE records for region %s" %
(len(unique_age_records), str(region_object)))
for age_record in unique_age_records:
thread_logger.info(str(age_record))
sv_types = list(
set([
age_record.contig.sv_type
for age_record in unique_age_records
]))
if len(sv_types) != 1:
thread_logger.error(
"Some problem. Mixed SV types for this interval %s" %
(str(sv_types)))
else:
sv_type = sv_types[0]
thread_logger.info("Processing region of type %s" % sv_type)
breakpoints, info_dict = process_age_records(
unique_age_records,
sv_type=sv_type,
pad=pad,
dist_to_expected_bp=dist_to_expected_bp,
min_del_subalign_len=min_del_subalign_len,
min_inv_subalign_len=min_inv_subalign_len,
age_window=age_window,
sc_locations=sc_locations)
bedtools_fields = matching_interval.fields
if len(breakpoints) == 1 and sv_type == "INS":
bedtools_fields += map(str, [
breakpoints[0][0], breakpoints[0][0] + 1,
breakpoints[0][1], breakpoints[0][2]
])
elif len(breakpoints) == 2 and (sv_type
in ["DEL", "INV", "DUP"]):
bedtools_fields += map(
str, breakpoints + [breakpoints[1] - breakpoints[0]] +
["."])
else:
bedtools_fields += map(
str, [bedtools_fields[1], bedtools_fields[2], -1, "."])
bedtools_fields[3] += ";AS"
bedtools_fields.append(base64.b64encode(json.dumps(info_dict)))
thread_logger.info("Writing out fields %s" %
(str(bedtools_fields)))
bedtools_intervals.append(
pybedtools.create_interval_from_list(bedtools_fields))
if not keep_temp:
os.remove(ref_name)
except Exception as e:
thread_logger.error('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
if assembly_fasta:
assembly_fasta.close()
reference_fasta.close()
thread_logger.info("Writing %d intervals" % (len(bedtools_intervals)))
if bedtools_intervals:
breakpoints_bed = os.path.join(age_workdir,
"%d_breakpoints.bed" % myid)
pybedtools.BedTool(bedtools_intervals).saveas(breakpoints_bed)
return breakpoints_bed
def run_oases(assmebly_hash=DNV_HASH,
seq_1="",
seq_2="",
seq_u="",
seq_i="",
file_format=DNV_FORMAT,
read_type=DNV_READTYPE,
oases=OASES,
velvetg=VELVETG,
velveth=VELVETH,
oases_opts="",
velvetg_opts="",
velveth_opts="",
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running de novo assembly (OASES) for %s" % sample)
if seq_1 and seq_2:
for s1 in seq_1.split(","):
if not os.path.exists(s1):
logger.error("Aborting!")
raise Exception("No Mate 1 sequence file %s" % s1)
for s2 in seq_2.split(","):
if not os.path.exists(s2):
logger.error("Aborting!")
raise Exception("No Mate 2 sequence file %s" % s2)
seq_argument = "-separate %s %s" % (seq_1, seq_2)
elif seq_u:
seq_argument = seq_u
for su in seq_u.split(","):
if not os.path.exists(su):
logger.error("Aborting!")
raise Exception("No unpaired sequence file %s" % su)
elif seq_i:
seq_argument = seq_i
for sr in seq_i.split(","):
if not os.path.exists(seq_i):
logger.error("Aborting!")
raise Exception("No sra sequence file %s" % sr)
work_oases = os.path.join(workdir, "oases", sample)
create_dirs([work_oases])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase Oases work directory for %s" % sample
command = "rm -rf %s/*" % (work_oases)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
oases_log = os.path.join(work_oases, "oases.log")
oases_log_fd = open(oases_log, "w")
seq_argument = "-%s -%s %s " % (file_format, read_type, seq_argument)
msg = "velveth for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s %d %s %s" % (velveth, work_oases, assmebly_hash,
velveth_opts, seq_argument)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=oases_log_fd,
cmd_log=oases_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "velvetg for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s %s -read_trkg yes " % (velvetg, work_oases,
velvetg_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=oases_log_fd,
cmd_log=oases_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "oases for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s %s " % (oases, work_oases, oases_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=oases_log_fd,
cmd_log=oases_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_oases = os.path.join(outdir, "oases", sample)
create_dirs([out_oases])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/transcripts.fa" % work_oases):
command = "cp %s/transcripts.fa %s/transcripts.fa" % (work_oases,
out_oases)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=oases_log_fd,
cmd_log=oases_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
transcripts = ""
if os.path.exists("%s/transcripts.fa" % out_oases):
logger.info("Oases was successfull!")
logger.info("Output transcripts: %s/transcripts.fa" % out_oases)
transcripts = "%s/transcripts.fa" % out_oases
else:
logger.info("Oases failed!")
return transcripts
def run_afterqc(fqdir="",
r1_flag="",
r2_flag="",
start=0,
sample="",
afterqc_opts="",
workdir=None,
outdir=None,
timeout=TIMEOUT,
nthreads=1):
logger.info(
"Automatic Filtering, Trimming, Error Removing and Quality Control for fastq data for %s"
% sample)
work_qc = os.path.join(workdir, "qc")
create_dirs([work_qc])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase QC work directory for %s" % sample
command = "rm -rf %s/*" % (work_qc)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
qc_log = os.path.join(work_qc, "qc.log")
qc_log_fd = open(qc_log, "w")
#seq_argument="-%s -%s %s "%(file_format,read_type,seq_argument)
msg = "QC for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -d %s --read1_flag %s --read2_flag %s -g %s -b %s -r %s %s" % (
"python", "/opt/AfterQC/after.py", fqdir, r1_flag, r2_flag,
work_qc + "/good/", work_qc + "/bad/", work_qc + "/qc_report/",
afterqc_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=qc_log_fd,
cmd_log=qc_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_qc = os.path.join(outdir, "qc")
create_dirs([out_qc])
msg = "Copy qc html report to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
qclist = filter(lambda x: x.endswith(("html")),
os.listdir("%s/qc_report/" % work_qc))
if len(qclist) > 0:
#command=" && ".join(map(lambda x: "cp %s/qc_report/%s %s/" % (work_qc, x, out_qc), qclist))
command = "cp %s %s/" % (" ".join(
map(lambda x: "%s/qc_report/%s" %
(work_qc, x), qclist)), out_qc)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=qc_log_fd,
cmd_log=qc_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
def run_lordec(kmer=23,
solid=3,
long="",
short="",
lordec=LORDEC,
lordec_opts="",
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running long read error correction (LoRDEC) for %s" % sample)
if not os.path.exists(long):
logger.error("Aborting!")
raise Exception("No long read sequence file %s" % long)
if not os.path.exists(short):
logger.error("Aborting!")
raise Exception("No short read sequence file %s" % short)
work_lordec = os.path.join(workdir, "lordec", sample)
create_dirs([work_lordec])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase LoRDEC work directory for %s" % sample
command = "rm -rf %s/*" % (work_lordec)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
lordec_log = os.path.join(work_lordec, "lordec.log")
lordec_log_fd = open(lordec_log, "w")
ksps = ""
if "-T " not in lordec_opts:
lordec_opts += " -T %d" % nthreads
msg = "LoRDEC for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -k %d -s %d -i %s -2 %s -O %s -o %s/long_corrected.fa" % (
lordec, lordec_opts, kmer, solid, long, short, work_lordec,
work_lordec)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=lordec_log_fd,
cmd_log=lordec_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_lordec = os.path.join(outdir, "lordec", sample)
create_dirs([out_lordec])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/long_corrected.fa" % work_lordec):
command = "cp %s/long_corrected.fa %s/long_corrected.fa" % (
work_lordec, out_lordec)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=lordec_log_fd,
cmd_log=lordec_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
corrected = ""
if os.path.exists("%s/long_corrected.fa" % out_lordec):
logger.info("LoRDEC was successfull!")
logger.info("Output corrected reads: %s/long_corrected.fa" %
out_lordec)
corrected = "%s/long_corrected.fa" % out_lordec
else:
logger.info("LoRDEC failed!")
return corrected
def annotate_ver2(sample="",
start=0,
nettype="0",
dcutt="0.5",
dcutnt="0.45",
orgtype="0",
minsglen="10",
trunc="70",
orgname="",
blastp_opts="",
evalue="",
msa="",
workdir=None,
outdir=None,
timeout=TIMEOUT,
nthreads=1):
logger.info("Annotation for precursor proteins of polypeptides for %s" %
sample)
work_annot = os.path.join(workdir, "annotation")
create_dirs([work_annot])
work_msalign = os.path.join(workdir, "msalign")
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase annotation work directory for %s" % sample
command = "rm -rf %s/*" % (work_annot)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
annot_log = os.path.join(work_annot, "annotation.log")
annot_log_fd = open(annot_log, "w")
msg = "Predicting the presence and location of signal peptide cleavage sites in amino acid sequences for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = (
"casperjs /opt/Auxtools/spider/signalP.js --fasta=%(msdir)s/%(sam)s-sequence.fa "
"--outfile=%(atdir)s/%(sam)s-signalP.txt --minlen=\"%(mlen)s\" "
"--method=\"%(nettype)s\" --orgtype=\"%(orgtype)s\" "
"--dcut=\"user\" --notm=%(dcutnt)s --tm=%(dcutt)s --trunc=%(trunc)s"
) % {
'msdir': work_msalign,
'atdir': work_annot,
'sam': sample,
'mlen': minsglen,
'nettype': nettype,
'orgtype': orgtype,
'dcutt': dcutt,
'dcutnt': dcutnt,
'trunc': trunc
}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Annotating precursor protein function for %s" % sample
sma3sfa = filter(lambda x: x.endswith(("fa", "fasta")),
os.listdir(SMA3SDB_DIR))
sma3sat = filter(lambda x: x.endswith(("annot")), os.listdir(SMA3SDB_DIR))
sma3sfagz = filter(lambda x: x.endswith(("fa.gz", "fasta.gz")),
os.listdir(SMA3SDB_DIR))
sma3satgz = filter(lambda x: x.endswith(("annot.gz")),
os.listdir(SMA3SDB_DIR))
if (len(sma3sfa) == 1 and len(sma3sat) == 1):
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = (
"mkdir -p %(atdir)s/sma3s/ && "
"ln -sf /data/%(ssdir)s/%(dbfa)s %(atdir)s/sma3s/%(dbfa)s && "
"ln -sf /data/%(ssdir)s/%(dbat)s %(atdir)s/sma3s/%(dbat)s && "
"cp %(msdir)s/%(sam)s-sequence.fa %(atdir)s/sma3s/ &&"
"cd %(atdir)s/sma3s/ && "
"perl /opt/Auxtools/sma3s_v2.pl -i %(sam)s-sequence.fa -d %(dbfa)s -go -goslim -p 0.00001 -num_threads %(nthreads)s && "
"cd -") % {
'msdir': work_msalign,
'atdir': work_annot,
'ssdir': SMA3SDB_DIR,
'dbfa': sma3sfa[0],
'dbat': sma3sat[0],
'sam': sample,
'nthreads': nthreads
}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
elif (len(sma3sfagz) == 1 and len(sma3satgz) == 1):
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = (
"mkdir -p %(atdir)s/sma3s/ && "
"gzip -dc %(ssdir)s/%(dbatgz)s > %(atdir)s/sma3s/%(dbat)s && "
"gzip -dc %(ssdir)s/%(dbfagz)s > %(atdir)s/sma3s/%(dbfa)s && "
"cp %(msdir)s/%(sam)s-sequence.fa %(atdir)s/sma3s/ &&"
"cd %(atdir)s/sma3s/ && "
"perl /opt/Auxtools/sma3s_v2.pl -i %(sam)s-sequence.fa -d %(dbfa)s -go -goslim -p 0.00001 -num_threads %(nthreads)s && "
"cd -") % {
'msdir': work_msalign,
'atdir': work_annot,
'ssdir': SMA3SDB_DIR,
'dbfagz': sma3sfagz[0],
'dbatgz': sma3satgz[0],
'dbfa': os.path.splitext(sma3sfagz[0])[0],
'dbat': os.path.splitext(sma3satgz[0])[0],
'sam': sample,
'nthreads': nthreads
}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
else:
logger.warn(
"Two Sma3s database files(uniref90.annot and uniref90.fasta) did not exist in the '%s' directory! Please obtain it from http://www.bioinfocabd.upo.es/sma3s/db/"
% (BLASTDB_DIR))
logger.warn("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Multiple sequence alignment for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = (
"python /opt/Auxtools/msa.py -i %(msdir)s/%(sam)s-sequence.fa "
"-m %(msa)s -o %(atdir)s/%(sam)s-msa.html ") % {
'msdir': work_msalign,
'atdir': work_annot,
'sam': sample,
'msa': msa
}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Venom annotation for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = (
"python /opt/Auxtools/venomkb/venomkb_annot.py -i %(msdir)s/%(sam)s-sequence.fa "
"-c /opt/Auxtools/venomkb/venomkb_proteins_06272017.json.gz -o %(atdir)s/%(sam)s "
) % {
'msdir': work_msalign,
'atdir': work_annot,
'sam': sample
}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Similar sequence blast for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
falist = filter(lambda x: x.endswith(("fa", "fasta")),
os.listdir(BLASTDB_DIR))
if (len(falist) == 0):
logger.warn(
"There is no blast database file(*.fa or *.fasta) in the '%s' directory!"
% (BLASTDB_DIR))
logger.warn("Skipping step %d: %s" % (step, msg))
if (len(falist) > 1):
logger.warn(
"Only one blast database file is allowed in the '%s' directory!"
% (BLASTDB_DIR))
logger.warn("Skipping step %d: %s" % (step, msg))
dbext = filter(lambda x: x.endswith(("phr", "pin", "psq")),
os.listdir("./config/blastdb/"))
if (len(dbext) < 3): #check db had been builded.
cmd_chip1 = "makeblastdb -dbtype prot -in %(path)s/%(db)s -out %(path)s/%(db)s && \
" % {
'path': BLASTDB_DIR,
'db': falist[0]
}
else:
cmd_chip1 = ""
cmd_chip2 = (
"blastp -db %(path)s/%(db)s -num_threads %(nthreads)s -query %(msdir)s/%(sam)s-sequence.fa "
"-out %(atdir)s/%(sam)s.asn -outfmt 11 -evalue %(evalue)s %(opts)s && "
"blast_formatter -archive %(atdir)s/%(sam)s.asn -outfmt 0 > %(atdir)s/%(sam)s-pairwise.txt && "
"blast_formatter -archive %(atdir)s/%(sam)s.asn -outfmt '7 qaccver saccver pident length mismatch gapopen qstart qend sstart send evalue bitscore stitle' > %(atdir)s/%(sam)s-tabular.txt && "
"python /opt/Auxtools/BlasterJS/src/blast2html.py -i %(atdir)s/%(sam)s-pairwise.txt "
"-o %(atdir)s/blast_html/") % {
'path': BLASTDB_DIR,
'db': falist[0],
'msdir': work_msalign,
'atdir': work_annot,
'sam': sample,
'orgname': orgname,
'evalue': evalue,
'opts': blastp_opts,
'nthreads': nthreads
}
command = cmd_chip1 + cmd_chip2
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_annot = os.path.join(outdir, "annotation")
create_dirs([out_annot])
msg = "Copy annotation result to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/%s-signalP.txt" % (work_annot, sample)):
command = (
"cp %(indir)s/%(sam)s-msa.html %(indir)s/%(sam)s-venom.tsv "
"%(indir)s/%(sam)s-signalP.txt %(outdir)s/") % {
"indir": work_annot,
"sam": sample,
"outdir": out_annot
}
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=annot_log_fd,
cmd_log=annot_log,
msg=msg,
timeout=timeout)
if os.path.exists("%s/blast_html" % (work_annot)):
copy_and_overwrite("%s/blast_html" % (work_annot),
"%s/blast_html/" % (out_annot))
tmpin = "%s/%s-tabular.txt" % (work_annot, sample)
tmpout = "%s/%s-tabular.txt" % (out_annot, sample)
if os.path.exists(tmpin):
copyfile(tmpin, tmpout)
tsvlist = filter(lambda x: x.endswith(("tsv")),
os.listdir(os.path.join(work_annot, "sma3s")))
logger.info(tsvlist)
summary = filter(lambda x: x.endswith(("summary.tsv")), tsvlist)[0]
tsvtab = filter(lambda x: not x.endswith(("summary.tsv")), tsvlist)[0]
tmpin = "%s" % os.path.join(work_annot, "sma3s", summary)
tmpout = "%s/%s-sma3s-summary.tsv" % (out_annot, sample)
if os.path.exists(tmpin):
copyfile(tmpin, tmpout)
tmpin = "%s" % os.path.join(work_annot, "sma3s", tsvtab)
tmpout = "%s/%s-sma3s-table.tsv" % (out_annot, sample)
if os.path.exists(tmpin):
copyfile(tmpin, tmpout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
return os.EX_OK
def run_comet(input="", longest=False, spectrum="",
start=0, sample= "", nthreads=1,
workdir=None, outdir=None, timeout=TIMEOUT):
logger.info("Running mass spectra alignment (Comet) for %s"%sample)
work_msalign=os.path.join(workdir,"msalign")
create_dirs([work_msalign])
step=0
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
msg = "Erase msalign work directory for %s"%sample
command="rm -rf %s/*" % (
work_msalign)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step+=1
msalign_log = os.path.join(work_msalign, "msalign.log")
msalign_log_fd = open(msalign_log, "w")
#determine wether the fasta is nucleotide or amino acid format.
tmpfile=open(input)
tmpline=tmpfile.readlines()[1] #get second line
is_na=True
if len(set(tmpline.strip()))>4:
is_na=False
msg = "Run PGA database creator for %s"%sample
if start<=step and is_na:
logger.info("--------------------------STEP %s--------------------------"%step)
command="Rscript /opt/Auxtools/run_dbcreator.R %s %s %s %s" % (
input, longest, work_msalign, sample)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
elif start<=step and not is_na:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("mkdir -p %(wk)s/database/ && cp %(db)s %(wk)s/database/%(sam)s.ntx.fasta") % {
'db': input,
'wk' : work_msalign,
'sam' : sample
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Run Comet for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("/opt/comet.2018012.linux.exe -Pconfig/par/comet.params -N%(dir)s/%(sam)s "
"-D%(dir)s/database/%(sam)s.ntx.fasta %(spectrum)s ") % {
'spectrum': spectrum,
'dir':work_msalign,
'sam':sample
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Tidy identification result and get precursor protein for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("Rscript /opt/Auxtools/comet_fdr.R %(dir)s/%(sam)s %(dir)s/database/%(sam)s.ntx.fasta ") % {
'dir': work_msalign,
'sam': sample
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
out_msalign=os.path.join(outdir,"msalign")
out_database=os.path.join(outdir,"database")
create_dirs([out_msalign, out_database])
msg="Copy novel sequence database and MS identification result(s) to output directory for %s."%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/%s-pepSummary.tsv"% (work_msalign, sample)):
command = "cp %(dir)s/%(sam)s-pepSummary.tsv %(dir)s/%(sam)s-psmSummary.tsv %(dir)s/%(sam)s-sequence.fa %(out)s/"%{
'dir': work_msalign,
'sam': sample,
'out': out_msalign
}
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
if os.path.exists("%s/database/%s.ntx.fasta"% (work_msalign, sample)):
command = "cp %(dir)s/database/%(sam)s.ntx.fasta %(out)s/" % {
'dir': work_msalign,
'sam': sample,
'out': out_database
}
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
return os.EX_OK
def run_msgfplus(input="", longest=False,
spectrum="",instrument="3",enzyme="0",decoy="1", fragid="0",
pretol="20ppm",minlen=6,maxlen=50,modfile="",ntt="0",
start=0, sample= "", nthreads=1,
msgfplus_opts="", max_mem="10G",
workdir=None, outdir=None, timeout=TIMEOUT):
logger.info("Running mass spectra alignment (MSGFPlus) for %s"%sample)
work_msalign=os.path.join(workdir,"msalign")
create_dirs([work_msalign])
step=0
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
msg = "Erase msalign work directory for %s"%sample
command="rm -rf %s/*" % (
work_msalign)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step+=1
msalign_log = os.path.join(work_msalign, "msalign.log")
msalign_log_fd = open(msalign_log, "w")
#determine wether the fasta is nucleotide or amino acid format.
tmpfile=open(input)
tmpline=tmpfile.readlines()[1] #get second line
is_na=True
if len(set(tmpline.strip()))>4:
is_na=False
msg = "Run PGA database creator for %s"%sample
if start<=step and is_na:
logger.info("--------------------------STEP %s--------------------------"%step)
command="Rscript /opt/Auxtools/run_dbcreator.R %s %s %s %s" % (
input, longest, work_msalign, sample)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
elif start<=step and not is_na:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("mkdir -p %(wk)s/database/ && cp %(db)s %(wk)s/database/%(sam)s.ntx.fasta") % {
'db': input,
'wk' : work_msalign,
'sam' : sample
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Run MSGFPlus for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("java -jar /opt/MSGFPlus/MSGFPlus.jar -s %(spectrum)s "
"-o %(dir)s/%(sam)s.mzid -d %(dir)s/database/%(sam)s.ntx.fasta -m %(fragid)s "
"-t %(pretol)s -inst %(inst)s -e %(eyz)s -ntt %(ntt)s -tda %(tda)s -minLength %(minl)s "
"-maxLength %(maxl)s -thread %(th)s -mod %(modfile)s") % {
'spectrum': spectrum,
'dir':work_msalign,
'sam':sample,
"pretol": pretol,
"inst": instrument,
"eyz": enzyme,
"tda": decoy,
"ntt": ntt,
"minl": minlen,
"maxl": maxlen,
'modfile': modfile,
"fragid": fragid,
'th': nthreads
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Converts MS-GF+ output (.mzid) into the tsv format (.tsv) for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("java -cp /opt/MSGFPlus/MSGFPlus.jar edu.ucsd.msjava.ui.MzIDToTsv "
"-i %(dir)s/%(sam)s.mzid -o %(dir)s/%(sam)s-rawSummary.tsv "
"-showQValue 1 -showDecoy 0 -unroll 1") % {
'dir':work_msalign,
'sam':sample,
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Tidy identification result and get precursor protein for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command=("python /opt/Auxtools/fasta_preparation.py -i %(dir)s/%(sam)s-rawSummary.tsv "
"-d %(dir)s/database/%(sam)s.ntx.fasta -o %(dir)s/%(sam)s") % {
'dir': work_msalign,
'sam': sample
}
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True, env_dict={"OMP_NUM_THREADS":str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
out_msalign=os.path.join(outdir,"msalign")
out_database=os.path.join(outdir,"database")
create_dirs([out_msalign, out_database])
msg="Copy novel sequence database and MS identification result(s) to output directory for %s."%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/%s-pepSummary.tsv"% (work_msalign, sample)):
command = "cp %(dir)s/%(sam)s-pepSummary.tsv %(dir)s/%(sam)s-psmSummary.tsv %(dir)s/%(sam)s-sequence.fa %(out)s/"%{
'dir': work_msalign,
'sam': sample,
'out': out_msalign
}
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
if os.path.exists("%s/database/%s.ntx.fasta"% (work_msalign, sample)):
command = "cp %(dir)s/database/%(sam)s.ntx.fasta %(out)s/" % {
'dir': work_msalign,
'sam': sample,
'out': out_database
}
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=msalign_log_fd, cmd_log=msalign_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
return os.EX_OK
def run_stringtie(alignment_bam="",
ref_gtf="",
stringtie_opts="",
stringtie=STRINGTIE,
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running transcriptome reconstruction (StringTie) for %s" %
sample)
if not os.path.exists(alignment_bam):
logger.error("Aborting!")
raise Exception("No input alignment BAM file %s" % alignment_bam)
work_stringtie = "%s/stringtie/%s/" % (workdir, sample)
create_dirs([work_stringtie])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase StringTie work directory for %s" % sample
command = "rm -rf %s/*" % (work_stringtie)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
stringtie_log = os.path.join(work_stringtie, "stringtie.log")
stringtie_log_fd = open(stringtie_log, "w")
if ref_gtf:
if not os.path.exists(ref_gtf):
logger.error("Aborting!")
raise Exception("No reference GTF file %s" % ref_gtf)
if ref_gtf:
stringtie_opts += " -G %s" % ref_gtf
if "-p " not in stringtie_opts:
stringtie_opts += " -p %d" % nthreads
msg = "StringTie for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s %s -o %s/transcripts.gtf -A %s/gene_abund.tab -v" % (
stringtie, alignment_bam, stringtie_opts, work_stringtie,
work_stringtie)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=stringtie_log_fd,
cmd_log=stringtie_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_stringtie = os.path.join(outdir, "stringtie", sample)
create_dirs([out_stringtie])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/transcripts.gtf"%work_stringtie) and \
os.path.exists("%s/gene_abund.tab"%work_stringtie):
command = "cp %s/transcripts.gtf %s/transcripts.gtf" % (
work_stringtie, out_stringtie)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=stringtie_log_fd,
cmd_log=stringtie_log,
msg=msg,
timeout=timeout)
command = "cp %s/gene_abund.tab %s/gene_abund.tab" % (
work_stringtie, out_stringtie)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=stringtie_log_fd,
cmd_log=stringtie_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
transcripts = ""
abundances = ""
if os.path.exists("%s/transcripts.gtf"%out_stringtie) and \
os.path.exists("%s/gene_abund.tab"%out_stringtie):
logger.info("StringTie was successfull!")
logger.info("Output isoforms: %s/transcripts.gtf" % out_stringtie)
logger.info("Output expressions: %s/gene_abund.tab" % out_stringtie)
transcripts = "%s/transcripts.gtf" % out_stringtie
abundances = "%s/gene_abund.tab" % out_stringtie
else:
logger.info("StringTie failed!")
return transcripts, abundances
def run_trinity(seq_1="",
seq_2="",
seq_u="",
start=0,
sample="",
nthreads=1,
trinity_opts="",
max_mem="20G",
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running de novo assembly (TRINITY) for %s" % sample)
#dirname="trinity_"+sample #Triniy's output directory must contain the word 'trinity' as a safety precaution
work_trinity = os.path.join(workdir, "trinity")
create_dirs([work_trinity])
#check the fq
if seq_1 and seq_2:
for s1 in seq_1.split(","):
if not os.path.exists(s1):
logger.error("Aborting!")
raise Exception("No Mate 1 sequence file %s" % s1)
if not s1.endswith(".fq.gz"):
logger.error(
"Aborting! Please ensure the suffix of fastq files is <*>.fq.gz"
)
raise Exception("Fastq format error %s" % s1)
for s2 in seq_2.split(","):
if not os.path.exists(s2):
logger.error("Aborting!")
raise Exception("No Mate 2 sequence file %s" % s2)
if not s2.endswith(".fq.gz"):
logger.error(
"Aborting! Please ensure the suffix of fastq files is <*>.fq.gz"
)
raise Exception("Fastq format error %s" % s2)
seq_argument = "--left %s --right %s" % (seq_1, seq_2)
'''
cor_1=chainMap(seq_1.split(",")).
map(lambda x: os.path.basename(x)).
map(lambda x: re.sub(r'(.+)\.(fq|fastq)(\.gz)?', r'\1', x)).
map(lambda x: work_trinity + "/corfq/"+ x +".cor.fq.gz")
scor_1=",".join(cor_1)
cor_2=chainMap(seq_2.split(",")).
map(lambda x: os.path.basename(x)).
map(lambda x: re.sub(r'(.+)\.(fq|fastq)(\.gz)?', r'\1', x)).
map(lambda x: work_trinity + "/corfq/"+ x +".cor.fq.gz")
scor_2=",".join(cor_2)
'''
elif seq_u:
for su in seq_u.split(","):
if not os.path.exists(su):
logger.error("Aborting!")
raise Exception("No unpaired sequence file %s" % su)
if not su.endswith(".fq.gz"):
logger.error(
"Aborting! Please ensure the suffix of fastq files is <*>.fq.gz"
)
raise Exception("Fastq format error %s" % su)
seq_argument = "--single %s" % (seq_u)
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase Trinity work directory for %s" % sample
#Trinity's running status is stored with the hidden files. It's necessary to delete these hidden file before the rerun of Trinity. Note that '*' couldn't match the hidden files.
command = "rm -rf %(wk)s/* %(wk)s/.*" % {'wk': work_trinity}
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
trinity_log = os.path.join(work_trinity, "trinity.log")
trinity_log_fd = open(trinity_log, "w")
#seq_argument="-%s -%s %s "%(file_format,read_type,seq_argument)
msg = "Run Trinity for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "Trinity --seqType fq --max_memory %s --CPU %s --output %s %s %s" % (
max_mem, nthreads, work_trinity, seq_argument, trinity_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True,
env_dict={"OMP_NUM_THREADS": str(nthreads)})
retcode = cmd.run(cmd_log_fd_out=trinity_log_fd,
cmd_log=trinity_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_trinity = os.path.join(outdir, "trinity")
create_dirs([out_trinity])
msg = "Copy trinity transcripts to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/Trinity.fasta" % work_trinity):
command = "cp %s/Trinity.fasta %s/" % (work_trinity, out_trinity)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=trinity_log_fd,
cmd_log=trinity_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
return os.EX_OK
def run_idpfusion(alignment="",
short_junction="",
long_alignment="",
mode_number=0,
short_fasta="",
long_fasta="",
ref_genome="",
ref_all_gpd="",
ref_gpd="",
uniqueness_bedgraph="",
genome_bowtie2_idx="",
transcriptome_bowtie2_idx="",
read_length=100,
idpfusion_cfg="",
idpfusion=IDPFUSION,
samtools=SAMTOOLS,
gmap=GMAP,
gmap_idx="",
star_dir=STAR_DIR,
bowtie2_dir=BOWTIE2_DIR,
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running long read fusion Detection (IDP-fusion) for %s" %
sample)
if not os.path.exists(alignment):
logger.error("Aborting!")
raise Exception("No input short read alignment BAM/SAM file %s" %
alignment)
if not os.path.exists(short_junction):
logger.error("Aborting!")
raise Exception("No input short read junction BED file %s" %
short_junction)
if idpfusion_cfg:
if not os.path.exists(idpfusion_cfg):
logger.error("Aborting!")
raise Exception("No input .cfg file %s" % idpfusion_cfg)
if mode_number > 0:
start = 4
work_idpfusion = "%s/idpfusion/%s/" % (workdir, sample)
create_dirs([work_idpfusion])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase IDP-fusion work directory for %s" % sample
command = "rm -rf %s/*" % (work_idpfusion)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
idpfusion_log = os.path.join(work_idpfusion, "idpfusion.log")
idpfusion_log_fd = open(idpfusion_log, "w")
msg = "converting BAM to SAM for %s" % sample
logger.info("--------------------------STEP %s--------------------------" %
step)
if start <= step:
if alignment.endswith('.bam'):
command = "%s view -h -o %s/alignments.sam %s " % (
samtools, work_idpfusion, alignment)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idpfusion_log_fd,
cmd_log=idpfusion_log,
msg=msg,
timeout=timeout)
alignment = "%s/alignments.sam" % (work_idpfusion)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Fix soft-clipped reads in SAM for %s" % sample
logger.info("--------------------------STEP %s--------------------------" %
step)
if start <= step:
logger.info("Task: %s" % msg)
corrected_alignment = "%s/alignments_corrected.sam" % (work_idpfusion)
with open(alignment, "r") as csv_file_i:
with open(corrected_alignment, "w") as csv_file_o:
spamreader = csv.reader(csv_file_i,
delimiter='\t',
quotechar='|')
spamwriter = csv.writer(csv_file_o,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
for row in spamreader:
if row[0][0] == "@":
spamwriter.writerow(row)
continue
if row[5] == "*":
continue
if "S" in row[5]:
cigartuple = cigarstring_to_tuple(row[5])
if cigartuple[0][0] == 4:
row[9] = row[9][cigartuple[0][1]:]
row[10] = row[10][cigartuple[0][1]:]
cigartuple = cigartuple[1:]
if cigartuple[-1][0] == 4:
row[9] = row[9][:-cigartuple[-1][1]]
row[10] = row[10][:-cigartuple[-1][1]]
cigartuple = cigartuple[:-1]
row[5] = "".join([
"%d%s" % (x[1], CIGAR_OP_DICT_rev[x[0]])
for x in cigartuple
])
spamwriter.writerow(row)
alignment = corrected_alignment
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Fix junction bed for %s" % sample
logger.info("--------------------------STEP %s--------------------------" %
step)
if start <= step:
logger.info("Task: %s" % msg)
corrected_junction = "%s/splicesites_corrected.bed" % (work_idpfusion)
with open(short_junction, "r") as csv_file_i:
with open(corrected_junction, "w") as csv_file_o:
spamreader = csv.reader(csv_file_i,
delimiter='\t',
quotechar='|')
spamwriter = csv.writer(csv_file_o,
delimiter='\t',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
for row in spamreader:
if len(row) < 4:
spamwriter.writerow(row)
continue
if "]" in row[3]:
spamwriter.writerow(row)
continue
row[3] = "(2)[2_2](2/0)"
spamwriter.writerow(row)
short_junction = corrected_junction
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Preparing run.cfg for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
logger.info("Task: %s" % msg)
if idpfusion_cfg:
msg = "copy IDP-fusion .cfg file for %s" % sample
command = "cp %s %s/run.cfg" % (idpfusion_cfg, work_idpfusion)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idpfusion_log_fd,
cmd_log=idpfusion_log,
msg=msg,
timeout=timeout)
else:
f = open("%s/run.cfg" % work_idpfusion, 'w')
f.close()
cgf_dict = {}
with open("%s/run.cfg" % work_idpfusion, 'r') as cfg_file:
for line in cfg_file:
line = line.strip()
if line == '':
continue
if "=" in line and not line[0] == '#':
k, v = line.split("=")
k = k.strip()
v = v.strip()
cgf_dict[k] = v
with open("%s/run.cfg" % work_idpfusion, 'w') as cfg_file:
for k, v in cgf_dict.iteritems():
cfg_file.write("%s = %s \n" % (k, v))
if "temp_foldername" not in cgf_dict:
cfg_file.write("temp_foldername = %s/tmp/ \n" % work_idpfusion)
if "output_foldername" not in cgf_dict:
cfg_file.write("output_foldername = %s/out/ \n" %
work_idpfusion)
if "Nthread" not in cgf_dict:
cfg_file.write("Nthread = %d \n" % nthreads)
if "LR_psl_pathfilename" not in cgf_dict:
if long_alignment and os.path.exists(long_alignment):
cfg_file.write("LR_psl_pathfilename = %s \n" %
long_alignment)
if "LR_pathfilename" not in cgf_dict:
cfg_file.write("LR_pathfilename = %s \n" % long_fasta)
if "SR_sam_pathfilename" not in cgf_dict:
cfg_file.write("SR_sam_pathfilename = %s \n" % alignment)
if "SR_jun_pathfilename" not in cgf_dict:
cfg_file.write("SR_jun_pathfilename = %s \n" % short_junction)
if "SR_pathfilename" not in cgf_dict:
cfg_file.write("SR_pathfilename = %s \n" % short_fasta)
if "SR_aligner_choice" not in cgf_dict:
cfg_file.write("SR_aligner_choice = STAR \n")
if "star_path" not in cgf_dict:
cfg_file.write("star_path = %s \n" % star_dir)
if "gmap_executable_pathfilename" not in cgf_dict:
cfg_file.write("gmap_executable_pathfilename = %s \n" % gmap)
if "gmap_index_pathfoldername" not in cgf_dict:
cfg_file.write("gmap_index_pathfoldername = %s \n" % gmap_idx)
if "genome_bowtie2_index_pathfilename" not in cgf_dict:
cfg_file.write("genome_bowtie2_index_pathfilename = %s \n" %
genome_bowtie2_idx)
if "transcriptome_bowtie2_index_pathfilename" not in cgf_dict:
cfg_file.write(
"transcriptome_bowtie2_index_pathfilename = %s \n" %
transcriptome_bowtie2_idx)
if "allref_annotation_pathfilename" not in cgf_dict:
cfg_file.write("allref_annotation_pathfilename = %s \n" %
ref_all_gpd)
if "ref_annotation_pathfilename" not in cgf_dict:
cfg_file.write("ref_annotation_pathfilename = %s \n" % ref_gpd)
if "genome_pathfilename" not in cgf_dict:
cfg_file.write("genome_pathfilename = %s \n" % ref_genome)
if "estimator_choice" not in cgf_dict:
cfg_file.write("estimator_choice = MAP \n")
if "FPR" not in cgf_dict:
cfg_file.write("FPR = 0.1 \n")
if "Njun_limit" not in cgf_dict:
cfg_file.write("Njun_limit = 10 \n")
if "Niso_limit" not in cgf_dict:
cfg_file.write("Niso_limit = 20 \n")
if "L_exon_limit" not in cgf_dict:
cfg_file.write("L_exon_limit = 1700 \n")
if "L_min_intron" not in cgf_dict:
cfg_file.write("L_min_intron = 68 \n")
if "Bfile_Npt" not in cgf_dict:
cfg_file.write("Bfile_Npt = 50 \n")
if "Bfile_Nbin" not in cgf_dict:
cfg_file.write("Bfile_Nbin = 5 \n")
if "min_LR_overlap_len" not in cgf_dict:
cfg_file.write("min_LR_overlap_len = 100 \n")
if "LR_fusion_point_err_margin" not in cgf_dict:
cfg_file.write("LR_fusion_point_err_margin = 100 \n")
if "min_LR_fusion_point_search_distance" not in cgf_dict:
cfg_file.write("min_LR_fusion_point_search_distance = 20 \n")
if "uniq_LR_alignment_margin_perc" not in cgf_dict:
cfg_file.write("uniq_LR_alignment_margin_perc = 20 \n")
if "Niso_fusion_limit" not in cgf_dict:
cfg_file.write("Niso_fusion_limit = 1000 \n")
if "psl_type" not in cgf_dict:
cfg_file.write("psl_type = 0 \n")
if "read_length" not in cgf_dict:
cfg_file.write("read_length = %d \n" % read_length)
if "min_junction_overlap_len" not in cgf_dict:
cfg_file.write("min_junction_overlap_len = 10 \n")
if "I_refjun_isoformconstruction" not in cgf_dict:
cfg_file.write("I_refjun_isoformconstruction = 1 \n")
if "I_ref5end_isoformconstruction" not in cgf_dict:
cfg_file.write("I_ref5end_isoformconstruction = 1 \n")
if "I_ref3end_isoformconstruction" not in cgf_dict:
cfg_file.write("I_ref3end_isoformconstruction = 1 \n")
if "fusion_mode" not in cgf_dict:
cfg_file.write("fusion_mode = 1 \n")
if "uniqueness_bedGraph_pathfilename" not in cgf_dict:
cfg_file.write("uniqueness_bedGraph_pathfilename = %s \n" %
uniqueness_bedgraph)
if "exon_construction_junction_span" not in cgf_dict:
cfg_file.write("exon_construction_junction_span = 1 \n")
if "aligner_choice" not in cgf_dict:
cfg_file.write("aligner_choice = gmap \n")
if "aligner_choice" not in cgf_dict:
cfg_file.write("aligner_choice = gmap \n")
if "three_primer" not in cgf_dict:
cfg_file.write("three_primer = \n")
if "five_primer" not in cgf_dict:
cfg_file.write("five_primer = \n")
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
if star_dir:
os.environ["PATH"] += ":%s/" % star_dir
if bowtie2_dir:
os.environ["PATH"] += ":%s/" % bowtie2_dir
msg = "IDP-fusion for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s/run.cfg %d" % (idpfusion, work_idpfusion, mode_number)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idpfusion_log_fd,
cmd_log=idpfusion_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Convert transcript GPD file to GTF for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/out/isoform.gpd" % work_idpfusion):
sort_gpd("%s/out/isoform.gpd" % work_idpfusion,
"%s/out/isoform_sorted.gpd" % work_idpfusion)
command = "gpd2gtf.py \
%s/out/isoform_sorted.gpd %s/out/isoform.exp %s/out/isoform.gtf IDP" % (
work_idpfusion, work_idpfusion, work_idpfusion)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idpfusion_log_fd,
cmd_log=idpfusion_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_idpfusion = os.path.join(outdir, "idpfusion", sample)
create_dirs([out_idpfusion])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/out/fusion_report.tsv" % work_idpfusion):
command = "cp %s/out/fusion_report.tsv %s/fusion_report.tsv" % (
work_idpfusion, out_idpfusion)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idpfusion_log_fd,
cmd_log=idpfusion_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
fusions = ""
if os.path.exists("%s/fusion_report.tsv" % out_idpfusion):
logger.info("IDP-fusion was successfull!")
logger.info("Output fusions: %s/fusion_report.tsv" % out_idpfusion)
fusions = "%s/fusion_report.tsv" % out_idpfusion
else:
logger.info("IDP-fusion failed!")
return fusions
def run_hisat2(align_idx=None,
seq_1="",
seq_2="",
seq_u="",
seq_sra="",
ref_gtf="",
hisat2_opts="",
hisat2=HISAT2,
hisat2_sps=HISAT2_SPS,
samtools=SAMTOOLS,
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running alignment (HISAT2) for %s" % sample)
if not os.path.exists(align_idx + ".1.ht2"):
logger.error("Aborting!")
raise Exception("No HISAT index file %s.1.ht2" % align_idx)
if seq_1 and seq_2:
for s1 in seq_1.split(","):
if not os.path.exists(s1):
logger.error("Aborting!")
raise Exception("No Mate 1 sequence file %s" % s1)
for s2 in seq_2.split(","):
if not os.path.exists(s2):
logger.error("Aborting!")
raise Exception("No Mate 2 sequence file %s" % s2)
seq_argument = "-1 %s -2 %s" % (seq_1, seq_2)
elif seq_u:
seq_argument = "-U %s" % (seq_u)
for su in seq_u.split(","):
if not os.path.exists(su):
logger.error("Aborting!")
raise Exception("No unpaired sequence file %s" % su)
elif seq_sra:
seq_argument = "--sra-acc %s" % (seq_sra)
for sr in seq_sra.split(","):
if not os.path.exists(sr):
logger.error("Aborting!")
raise Exception("No sra sequence file %s" % sr)
work_hisat2 = os.path.join(workdir, "hisat2", sample)
create_dirs([work_hisat2])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase HISAT2 work directory for %s" % sample
command = "rm -rf %s/*" % (work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
hisat2_log = os.path.join(work_hisat2, "hisat2.log")
hisat2_log_fd = open(hisat2_log, "w")
ksps = ""
msg = "Prepare known-splicesites for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if ref_gtf:
if not os.path.exists(ref_gtf):
logger.error("Aborting!")
raise Exception("No reference GTF file %s" % ref_gtf)
else:
ksps = ref_gtf.strip() + "known-splicesite.txt"
if os.path.exists(ksps):
logger.info(
"Will use the precomputed %s as --known-splicesite-infile for HISAT2"
% ksps)
else:
msg = "compute --known-splicesite-infile for HISAT2"
ksps = os.path.join(work_hisat2, "known-splicesite.txt")
ksps_fd = open(ksps, "w")
command = "%s %s" % (hisat2_sps, ref_gtf)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command,
logger,
raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=ksps_fd,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
if "--dta " not in hisat2_opts:
hisat2_opts += " --dta"
if "--rg-id " not in hisat2_opts:
hisat2_opts += " --rg-id hisat2"
if "--rg " not in hisat2_opts:
hisat2_opts += " --rg SM:%s" % sample
if "--threads " not in hisat2_opts:
hisat2_opts += " --threads %d" % nthreads
if ksps:
hisat2_opts += " --known-splicesite-infile %s" % ksps
msg = "HISAT2 for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -x %s %s -S %s/alignments.sam --novel-splicesite-outfile %s/splicesites.tab" % (
hisat2, hisat2_opts, align_idx, seq_argument, work_hisat2,
work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "converting SAM to BAM for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s view -Su %s/alignments.sam -@ %d -o %s/alignments.bam" % (
samtools, work_hisat2, nthreads, work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "sorting BAM for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s sort -@ %d -T %s/alignments.sorted -o %s/alignments.sorted.bam %s/alignments.bam " % (
samtools, nthreads, work_hisat2, work_hisat2, work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Converting junctions to BED for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "hisat2_jun2bed.py %s/splicesites.tab %s/splicesites.bed " % (
work_hisat2, work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "Clean temp alignment files for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "rm %s/alignments.sam %s/alignments.bam" % (work_hisat2,
work_hisat2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_hisat2 = os.path.join(outdir, "hisat2", sample)
create_dirs([out_hisat2])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/alignments.sorted.bam"%work_hisat2) and \
os.path.exists("%s/splicesites.tab"%work_hisat2) and \
os.path.exists("%s/splicesites.bed"%work_hisat2):
command = "cp %s/alignments.sorted.bam %s/alignments.sorted.bam" % (
work_hisat2, out_hisat2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
command = "cp %s/splicesites.tab %s/splicesites.tab" % (
work_hisat2, out_hisat2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
command = "cp %s/splicesites.bed %s/splicesites.bed" % (
work_hisat2, out_hisat2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=hisat2_log_fd,
cmd_log=hisat2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
alignments_bam = ""
junctions_tab = ""
junctions_bed = ""
if os.path.exists("%s/alignments.sorted.bam" % out_hisat2):
logger.info("HISAT2 was successfull!")
logger.info("Output alignment: %s/alignments.sorted.bam" % out_hisat2)
logger.info("Output junction tab: %s/splicesites.tab" % out_hisat2)
logger.info("Output junction bed: %s/splicesites.bed" % out_hisat2)
alignments_bam = "%s/alignments.sorted.bam" % out_hisat2
junctions_tab = "%s/splicesites.tab" % out_hisat2
junctions_bed = "%s/splicesites.bed" % out_hisat2
else:
logger.info("HISAT2 failed!")
return alignments_bam, junctions_tab, junctions_bed
def run_giremi(alignment="", variant="",
strand_pos="", genes_pos="",
ref_genome="", knownsites="",
giremi_dir="", htslib_dir="",
samtools=SAMTOOLS, gatk=GATK,
java=JAVA, giremi_opts="", java_opts="",
VariantAnnotator_opts="",
start=0, sample= "", nthreads=1,
workdir=None, outdir=None, timeout=TIMEOUT):
logger.info("Running RNA editing detection (GIREMI) for %s"%sample)
if not os.path.exists(alignment):
logger.error("Aborting!")
raise Exception("No alignment file %s"%alignment)
if not os.path.exists(variant):
logger.error("Aborting!")
raise Exception("No variant VCF file %s"%variant)
if not os.path.exists(strand_pos):
logger.error("Aborting!")
raise Exception("No strand position BED file %s"%strand_pos)
if not os.path.exists(genes_pos):
logger.error("Aborting!")
raise Exception("No genes position BED file %s"%genes_pos)
if not os.path.exists(ref_genome):
logger.error("Aborting!")
raise Exception("No reference genome FASTA file %s"%ref_genome)
if not os.path.exists(knownsites):
logger.error("Aborting!")
raise Exception("No VCF knownsites file %s"%knownsites)
if giremi_dir:
if not os.path.exists(giremi_dir):
logger.error("Aborting!")
raise Exception("No GIREMI directory %s"%giremi_dir)
work_giremi=os.path.join(workdir,"giremi",sample)
create_dirs([work_giremi])
if nthreads>1:
if "-nt " not in VariantAnnotator_opts:
VariantAnnotator_opts += " -nt %d"%nthreads
if "-Xms" not in java_opts:
java_opts += " %s"%JAVA_XMS
if "-Xmx" not in java_opts:
java_opts += " %s"%JAVA_XMG
if "-Djava.io.tmpdir" not in java_opts:
java_opts += " -Djava.io.tmpdir=%s/javatmp/"%(work_giremi)
step=0
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
msg = "Erase GIREMI work directory for %s"%sample
command="rm -rf %s/*" % (
work_giremi)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg,timeout=timeout)
step+=1
giremi_log = os.path.join(work_giremi, "giremi.log")
giremi_log_fd = open(giremi_log, "w")
msg = "Sort BAM by name for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command="%s sort -n -@ %d %s %s/alignments.name_sorted" % (
samtools, nthreads, alignment, work_giremi)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Filter alignments mapped to multiple chromosoms for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
logger.info(msg)
filter_multi_chr_alignments("%s/alignments.name_sorted.bam"%work_giremi,"%s/alignments.chr_unique.bam"%work_giremi)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Sort BAM by pos for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command="%s sort -@ %d %s/alignments.chr_unique.bam %s/alignments.pos_sorted " % (
samtools, nthreads, work_giremi, work_giremi)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "GATK VariantAnnotator for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command="%s %s -jar %s -T VariantAnnotator -R %s -V %s -L %s -o %s/annotated.vcf --dbsnp %s %s" % (
java, java_opts, gatk, ref_genome,variant,variant,work_giremi,knownsites,VariantAnnotator_opts)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg="Find variant strands for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
logger.info(msg)
find_SNV_strands(strand_pos, genes_pos, "%s/annotated.vcf"%work_giremi, "%s/SNV_annotated.bed"%work_giremi)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
if htslib_dir:
if "LD_LIBRARY_PATH" in os.environ:
os.environ["LD_LIBRARY_PATH"] += ":%s/"%htslib_dir
else:
os.environ["LD_LIBRARY_PATH"] = htslib_dir
if giremi_dir:
os.environ["PATH"] += ":%s/"%giremi_dir
msg = "Run GIREMI for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command="cd %s && %s %s -f %s -l %s/SNV_annotated.bed -o %s/giremi_out.txt %s/alignments.pos_sorted.bam" % (
giremi_dir,GIREMI, giremi_opts, os.path.abspath(ref_genome), os.path.abspath(work_giremi), os.path.abspath(work_giremi),os.path.abspath(work_giremi))
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
if os.path.exists("%s/giremi_out.txt"%work_giremi) and not os.path.exists("%s/giremi_out.txt.res"%work_giremi):
msg="Identify N variants for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
logger.info(msg)
with open("%s/giremi_out.txt"%work_giremi) as csv_file_i:
spamreader = csv.reader(csv_file_i, delimiter='\t', quotechar='|')
with open("%s/N.bed"%work_giremi, 'wb') as csvfile_o:
spamwriter = csv.writer(csvfile_o, delimiter='\t',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
for row in spamreader:
if (row[5]=="N" or row[8]=="N"):
spamwriter.writerow([row[0],int(row[1])-1,row[1]])
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
cnt=len(pybedtools.BedTool("%s/N.bed"%work_giremi))
if cnt>0:
msg="Remove N variants for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
logger.info(msg)
pybedtools.BedTool("%s/SNV_annotated.bed"%work_giremi).intersect(
"%s/N.bed"%work_giremi,r=True, f=1, v=True).saveas("%s/SNV_annotated_filtered.bed"%work_giremi)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Rerun GIREMI for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/SNV_annotated_filtered.bed"%work_giremi):
command="cd %s && %s %s -f %s -l %s/SNV_annotated_filtered.bed -o %s/giremi_out.txt %s/alignments.pos_sorted.bam" % (
giremi_dir,GIREMI, giremi_opts, os.path.abspath(ref_genome), os.path.abspath(work_giremi), os.path.abspath(work_giremi),os.path.abspath(work_giremi))
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("No file %s/SNV_annotated_filtered.bed"%work_giremi)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
else:
step+=2
else:
step+=3
out_giremi=os.path.join(outdir,"giremi",sample)
create_dirs([out_giremi])
msg="Copy predictions to output directory for %s."%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/giremi_out.txt.res"%work_giremi):
command = "cp %s/giremi_out.txt.res %s/giremi_out.txt.res"%(
work_giremi, out_giremi)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=giremi_log_fd, cmd_log=giremi_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
edits = ""
if os.path.exists("%s/giremi_out.txt.res"%out_giremi):
logger.info("GIREMI was successfull!")
logger.info("Output edits: %s/giremi_out.txt.res"%out_giremi)
edits = "%s/giremi_out.txt.res"%out_giremi
else:
logger.info("GIREMI failed!")
return edits
def run_starlong(long="",
genome_dir="",
ref_gtf="",
starlong=STARLONG,
sam2psl=SAM2PSL,
samtools=SAMTOOLS,
starlong_opts="",
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running long read alignment (STARlong) for %s" % sample)
if not os.path.exists(genome_dir + "SAindex"):
logger.error("Aborting!")
raise Exception("No SAindex directory in %s" % genome_dir)
if long:
if not os.path.exists(long):
logger.error("Aborting!")
raise Exception("No long read sequence file %s" % long)
work_starlong = os.path.join(workdir, "starlong", sample)
create_dirs([work_starlong])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase STARlong work directory for %s" % sample
command = "rm -rf %s/*" % (work_starlong)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
starlong_log = os.path.join(work_starlong, "starlong.log")
starlong_log_fd = open(starlong_log, "w")
if ref_gtf:
if not os.path.exists(ref_gtf):
logger.error("Aborting!")
raise Exception("No reference GTF file %s" % ref_gtf)
if "--outSAMattrRGline" not in starlong_opts:
starlong_opts += " --outSAMattrRGline ID:STARlong SM:%s" % sample
if "--runThreadN " not in starlong_opts:
starlong_opts += " --runThreadN %d" % nthreads
if ref_gtf:
starlong_opts += " --sjdbGTFfile %s" % ref_gtf
for k, v in STARLONG_DEFAULTS.iteritems():
if k not in starlong_opts:
starlong_opts += " --%s %s" % (k, v)
msg = "STARlong for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s --runMode alignReads %s --genomeDir %s --readFilesIn %s --outFileNamePrefix %s/" % (
starlong, starlong_opts, genome_dir, long, work_starlong)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=starlong_log_fd,
cmd_log=starlong_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "converting SAM to PSL for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s -i %s/Aligned.out.sam -o %s/Aligned.out.psl" % (
sam2psl, work_starlong, work_starlong)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=starlong_log_fd,
cmd_log=starlong_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "converting SAM to BAM for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s view -Su %s/Aligned.out.sam -@ %d -o %s/Aligned.out.bam" % (
samtools, work_starlong, nthreads, work_starlong)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=starlong_log_fd,
cmd_log=starlong_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
#
# msg = "Clean temp alignment files for %s"%sample
# if start<=step:
# logger.info("--------------------------STEP %s--------------------------"%step)
# command="rm %s/Aligned.out.sam" % (work_starlong)
# command="bash -c \"%s\""%command
# cmd = TimedExternalCmd(command, logger, raise_exception=True)
# retcode = cmd.run(cmd_log_fd_out=starlong_log_fd, cmd_log=starlong_log, msg=msg, timeout=timeout)
# else:
# logger.info("Skipping step %d: %s"%(step,msg))
# step+=1
out_starlong = os.path.join(outdir, "starlong", sample)
create_dirs([out_starlong])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/Aligned.out.psl" % work_starlong):
command = "cp %s/Aligned.out.psl %s/Aligned.out.psl" % (
work_starlong, out_starlong)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=starlong_log_fd,
cmd_log=starlong_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
alignments_psl = ""
if os.path.exists("%s/Aligned.out.psl" % out_starlong):
logger.info("STARlong was successfull!")
logger.info("Output alignment: %s/Aligned.out.psl" % out_starlong)
alignments_psl = "%s/Aligned.out.psl" % out_starlong
else:
logger.info("STARlong failed!")
return alignments_psl
def run_idp(alignment="", short_junction="", long_alignment="",mode_number=0,
ref_genome="", ref_all_gpd="", ref_gpd="",read_length=100,
idp_cfg="", idp=IDP, samtools=SAMTOOLS,
start=0, sample= "", nthreads=1,
workdir=None, outdir=None, timeout=TIMEOUT):
logger.info("Running long-read transcriptome reconstruction (IDP) for %s"%sample)
if not os.path.exists(alignment):
logger.error("Aborting!")
raise Exception("No input short read alignment BAM/SAM file %s"%alignment)
if not os.path.exists(short_junction):
logger.error("Aborting!")
raise Exception("No input short read junction BED file %s"%short_junction)
if not os.path.exists(long_alignment):
logger.error("Aborting!")
raise Exception("No input long read alignment PSL file %s"%long_alignment)
if idp_cfg:
if not os.path.exists(idp_cfg):
logger.error("Aborting!")
raise Exception("No input .cfg file %s"%idp_cfg)
if mode_number>0:
start=4
work_idp="%s/idp/%s/"%(workdir,sample)
create_dirs([work_idp])
step=0
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
msg = "Erase IDP work directory for %s"%sample
command="rm -rf %s/*" % (
work_idp)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg,timeout=timeout)
step+=1
idp_log = os.path.join(work_idp, "idp.log")
idp_log_fd = open(idp_log, "w")
msg = "converting BAM to SAM for %s"%sample
logger.info("--------------------------STEP %s--------------------------"%step)
if start<=step:
if alignment.endswith('.bam'):
command = "%s view -h -o %s/alignments.sam %s " % (samtools,work_idp,alignment)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
alignment = "%s/alignments.sam"%(work_idp)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Preparing run.cfg for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if idp_cfg:
msg = "copy IDP .cfg file for %s"%sample
command="cp %s %s/run.cfg" % (
idp_cfg, work_idp)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
else:
f=open("%s/run.cfg"%work_idp, 'w')
f.close()
cgf_dict={}
with open("%s/run.cfg"%work_idp , 'r') as cfg_file:
for line in cfg_file:
line = line.strip()
if line=='':
continue
if "=" in line and not line[0]=='#' :
k,v=line.split("=")
k=k.strip()
v=v.strip()
cgf_dict[k]=v
with open("%s/run.cfg"%work_idp , 'w') as cfg_file:
for k,v in cgf_dict.iteritems():
cfg_file.write("%s = %s \n"%(k,v))
if "temp_foldername" not in cgf_dict:
cfg_file.write("temp_foldername = %s/tmp/ \n"%work_idp)
if "output_foldername" not in cgf_dict:
cfg_file.write("output_foldername = %s/out/ \n"%work_idp)
if "Nthread" not in cgf_dict:
cfg_file.write("Nthread = %d \n"%nthreads)
if "LR_psl_pathfilename" not in cgf_dict:
cfg_file.write("LR_psl_pathfilename = %s \n"%long_alignment)
if "SR_sam_pathfilename" not in cgf_dict:
cfg_file.write("SR_sam_pathfilename = %s \n"%alignment)
if "SR_jun_pathfilename" not in cgf_dict:
cfg_file.write("SR_jun_pathfilename = %s \n"%short_junction)
if "genome_pathfilename" not in cgf_dict:
cfg_file.write("genome_pathfilename = %s \n"%ref_genome)
if "allref_annotation_pathfilename" not in cgf_dict:
cfg_file.write("allref_annotation_pathfilename = %s \n"%ref_all_gpd)
if "ref_annotation_pathfilename" not in cgf_dict:
cfg_file.write("ref_annotation_pathfilename = %s \n"%ref_gpd)
if "estimator_choice" not in cgf_dict:
cfg_file.write("estimator_choice = MLE \n")
if "FPR" not in cgf_dict:
cfg_file.write("FPR = 0.05 \n")
if "Njun_limit" not in cgf_dict:
cfg_file.write("Njun_limit = 10 \n")
if "Niso_limit" not in cgf_dict:
cfg_file.write("Niso_limit = 100 \n")
if "aligner_choice" not in cgf_dict:
cfg_file.write("aligner_choice = gmap \n")
if "exon_construction_junction_span" not in cgf_dict:
cfg_file.write("exon_construction_junction_span = 1 \n")
if "read_length" not in cgf_dict:
cfg_file.write("read_length = %d \n"%read_length)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "IDP for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
command="%s %s/run.cfg %d" % (
idp, work_idp, mode_number)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
msg = "Convert transcript GPD file to GTF for %s"%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/out/isoform.gpd"%work_idp):
sort_gpd("%s/out/isoform.gpd"%work_idp,"%s/out/isoform_sorted.gpd"%work_idp)
command="gpd2gtf.py \
%s/out/isoform_sorted.gpd %s/out/isoform.exp %s/out/isoform.gtf IDP"%(work_idp,work_idp,work_idp)
command="bash -c \"%s\""%command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
out_idp=os.path.join(outdir,"idp",sample)
create_dirs([out_idp])
msg="Copy predictions to output directory for %s."%sample
if start<=step:
logger.info("--------------------------STEP %s--------------------------"%step)
if os.path.exists("%s/out/isoform.gtf"%work_idp) and \
os.path.exists("%s/out/isoform.exp"%work_idp):
command = "cp %s/out/isoform.gtf %s/isoform.gtf"%(
work_idp, out_idp)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
command = "cp %s/out/isoform.exp %s/isoform.exp"%(
work_idp, out_idp)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=idp_log_fd, cmd_log=idp_log, msg=msg, timeout=timeout)
else:
logger.info("Skipping step %d: %s"%(step,msg))
step+=1
transcripts = ""
abundances = ""
if os.path.exists("%s/isoform.gtf"%out_idp) and \
os.path.exists("%s/isoform.exp"%out_idp):
logger.info("IDP was successfull!")
logger.info("Output isoforms: %s/isoform.gtf"%out_idp)
logger.info("Output expressions: %s/isoform.exp"%out_idp)
transcripts = "%s/isoform.gtf"%out_idp
abundances = "%s/isoform.exp"%out_idp
else:
logger.info("IDP failed!")
return transcripts,abundances
def run_gatk(alignment="",
ref_genome="",
knownsites="",
picard=PICARD,
gatk=GATK,
java=JAVA,
java_opts="",
CleanSam=False,
IndelRealignment=False,
no_BaseRecalibrator=False,
AddOrReplaceReadGroups_opts="",
MarkDuplicates_opts="",
SplitNCigarReads_opts="",
RealignerTargetCreator_opts="",
IndelRealigner_opts="",
BaseRecalibrator_opts="",
PrintReads_opts="",
HaplotypeCaller_opts="",
VariantFiltration_opts="",
start=0,
sample="",
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running variant calling (GATK) for %s" % sample)
if not os.path.exists(alignment):
logger.error("Aborting!")
raise Exception("No alignment file %s" % alignment)
if not os.path.exists(ref_genome):
logger.error("Aborting!")
raise Exception("No reference genome FASTA file %s" % ref_genome)
work_gatk = os.path.join(workdir, "gatk", sample)
create_dirs([work_gatk])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase GATK work directory for %s" % sample
command = "rm -rf %s/*" % (work_gatk)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
gatk_log = os.path.join(work_gatk, "gatk.log")
gatk_log_fd = open(gatk_log, "w")
if "SO=" not in AddOrReplaceReadGroups_opts:
AddOrReplaceReadGroups_opts += " SO=coordinate"
if "RGLB=" not in AddOrReplaceReadGroups_opts:
AddOrReplaceReadGroups_opts += " RGLB=lib1"
if "RGPL=" not in AddOrReplaceReadGroups_opts:
AddOrReplaceReadGroups_opts += " RGPL=illumina"
if "RGPU=" not in AddOrReplaceReadGroups_opts:
AddOrReplaceReadGroups_opts += " RGPU=unit1"
if "RGSM=" not in AddOrReplaceReadGroups_opts:
AddOrReplaceReadGroups_opts += " RGSM=%s" % sample
if "CREATE_INDEX=" not in MarkDuplicates_opts:
MarkDuplicates_opts += " CREATE_INDEX=true"
if "VALIDATION_STRINGENCY=" not in MarkDuplicates_opts:
MarkDuplicates_opts += " VALIDATION_STRINGENCY=SILENT"
if "-rf " not in SplitNCigarReads_opts:
SplitNCigarReads_opts += " -rf %s" % GATK_SN_RF
if "-RMQF " not in SplitNCigarReads_opts:
SplitNCigarReads_opts += " -RMQF %d" % GATK_SN_RMQF
if "-RMQT " not in SplitNCigarReads_opts:
SplitNCigarReads_opts += " -RMQT %d" % GATK_SN_RMQT
if "-U " not in SplitNCigarReads_opts:
SplitNCigarReads_opts += " -U ALLOW_N_CIGAR_READS"
if knownsites:
if not os.path.exists(knownsites):
logger.error("Aborting!")
raise Exception("No VCF knownsites file %s" % knownsites)
if "--known " not in RealignerTargetCreator_opts:
RealignerTargetCreator_opts += " --known %s" % knownsites
if "-known " not in IndelRealigner_opts and "--knownAlleles " not in IndelRealigner_opts:
IndelRealigner_opts += " -known %s" % knownsites
if "-knownSites " not in BaseRecalibrator_opts:
BaseRecalibrator_opts += " -knownSites %s" % knownsites
if "-dontUseSoftClippedBases " not in HaplotypeCaller_opts:
HaplotypeCaller_opts += " -dontUseSoftClippedBases"
if "-stand_call_conf " not in HaplotypeCaller_opts:
HaplotypeCaller_opts += " -stand_call_conf %f" % GATK_HC_STANDCALLCONF
if "-stand_emit_conf " not in HaplotypeCaller_opts:
HaplotypeCaller_opts += " -stand_emit_conf %f" % GATK_HC_STANDEMITCONF
if "-window " not in VariantFiltration_opts:
VariantFiltration_opts += " -window %d" % GATK_VF_WINDOW
if "-cluster " not in VariantFiltration_opts:
VariantFiltration_opts += " -cluster %d" % GATK_VF_CLUSTER
if "-filterName FS " not in VariantFiltration_opts:
VariantFiltration_opts += " -filterName FS -filter 'FS > %f'" % GATK_VF_FSMIN
if "-filterName QD " not in VariantFiltration_opts:
VariantFiltration_opts += " -filterName QD -filter 'QD < %f'" % GATK_VF_QDMAX
if nthreads > 1:
if "-nct " not in BaseRecalibrator_opts:
BaseRecalibrator_opts += " -nct %d" % nthreads
if "-nct " not in PrintReads_opts:
PrintReads_opts += " -nct %d" % nthreads
if "-Xms" not in java_opts:
java_opts += " %s" % JAVA_XMS
if "-Xmx" not in java_opts:
java_opts += " %s" % JAVA_XMG
if "-Djava.io.tmpdir" not in java_opts:
java_opts += " -Djava.io.tmpdir=%s/javatmp/" % (work_gatk)
msg = "picard CleanSam for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if CleanSam:
command = "%s %s -cp %s picard.cmdline.PicardCommandLine CleanSam I=%s O=%s/alignments_clean.bam" % (
java, java_opts, picard, alignment, work_gatk)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
alignment = "%s/alignments_clean.bam" % work_gatk
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "picard AddOrReplaceReadGroups for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -cp %s picard.cmdline.PicardCommandLine AddOrReplaceReadGroups I=%s O=%s/rg_added_sorted.bam %s" % (
java, java_opts, picard, alignment, work_gatk,
AddOrReplaceReadGroups_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "picard MarkDuplicates for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -cp %s picard.cmdline.PicardCommandLine MarkDuplicates I=%s/rg_added_sorted.bam O=%s/dedupped.bam %s M=%s/output.metrics" % (
java, java_opts, picard, work_gatk, work_gatk, MarkDuplicates_opts,
work_gatk)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK SplitNCigarReads for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T SplitNCigarReads -R %s -I %s/dedupped.bam -o %s/split.bam %s" % (
java, java_opts, gatk, ref_genome, work_gatk, work_gatk,
SplitNCigarReads_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
split_bam = "%s/split.bam" % work_gatk
if IndelRealignment:
msg = "GATK RealignerTargetCreator for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T RealignerTargetCreator -R %s -I %s/split.bam -o %s/forIndelRealigner.intervals %s" % (
java, java_opts, gatk, ref_genome, work_gatk, work_gatk,
RealignerTargetCreator_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK IndelRealigner for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T IndelRealigner -R %s -I %s/split.bam -targetIntervals %s/forIndelRealigner.intervals -o %s/split_realigned.bam %s" % (
java, java_opts, gatk, ref_genome, work_gatk, work_gatk,
work_gatk, IndelRealigner_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
split_bam = "%s/split_realigned.bam" % work_gatk
else:
msg = "GATK RealignerTargetCreator for %s" % sample
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK IndelRealigner for %s" % sample
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
if not no_BaseRecalibrator:
msg = "GATK BaseRecalibrator for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T BaseRecalibrator -R %s -I %s -o %s/recal_data.table %s" % (
java, java_opts, gatk, ref_genome, split_bam, work_gatk,
BaseRecalibrator_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK PrintReads for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T PrintReads -R %s -I %s -BQSR %s/recal_data.table -o %s/bsqr.bam %s" % (
java, java_opts, gatk, ref_genome, split_bam, work_gatk,
work_gatk, PrintReads_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
split_bam = "%s/bsqr.bam" % work_gatk
else:
msg = "GATK BaseRecalibrator for %s" % sample
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK PrintReads for %s" % sample
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK HaplotypeCaller for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T HaplotypeCaller -R %s -I %s -o %s/variants.vcf %s" % (
java, java_opts, gatk, ref_genome, split_bam, work_gatk,
HaplotypeCaller_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "GATK VariantFiltration for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s %s -jar %s -T VariantFiltration -R %s -V %s/variants.vcf -o %s/variants_filtered.vcf %s" % (
java, java_opts, gatk, ref_genome, work_gatk, work_gatk,
VariantFiltration_opts)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_gatk = os.path.join(outdir, "gatk", sample)
create_dirs([out_gatk])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/variants_filtered.vcf" % work_gatk):
command = "cp %s/variants_filtered.vcf %s/variants_filtered.vcf" % (
work_gatk, out_gatk)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=gatk_log_fd,
cmd_log=gatk_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
variants = ""
if os.path.exists("%s/variants_filtered.vcf" % out_gatk):
logger.info("GATK was successfull!")
logger.info("Output variants: %s/variants_filtered.vcf" % out_gatk)
variants = "%s/variants_filtered.vcf" % out_gatk
else:
logger.info("GATK failed!")
return variants
def run_deseq2(quant_files="",
alignments="",
transcripts_gtfs="",
ref_gtf="",
featureCounts_opts="",
featureCounts=FEATURECOUNTS,
stringtie=STRINGTIE,
stringtie_merge_opts="",
mincount=DESeq2_MINCNT,
alpha=DESeq2_ALPHA,
R=R_CMD,
start=0,
samples=[],
nthreads=1,
workdir=None,
outdir=None,
timeout=TIMEOUT):
samples = map(lambda x: x.split(","), samples)
samples_txt = "-".join(map(lambda x: ",".join(x), samples))
logger.info("Running differential analysis (DESeq2) for %s" % samples_txt)
n_samples = len(samples)
n_replicates = map(len, samples)
use_quant = True
use_refgtf = False
if quant_files and ref_gtf:
if len(quant_files) != n_samples:
logger.error("Aborting!")
raise Exception(
"Number of input quantification files does not match the number of samples (%s != %s)"
% (len(quant_files), n_samples))
quant_files = map(lambda x: x.split(","), quant_files)
for i, q in enumerate(quant_files):
if len(q) != n_replicates[i]:
logger.error("Aborting!")
raise Exception(
"Number of input quantification replicate files does not match the number of replicates in %d%s sample (%s != %s)"
%
(i + 1, "st" if i > 0 else "th", len(q), n_replicates[i]))
for r in q:
if not os.path.exists(r):
logger.error("Aborting!")
raise Exception("No qantification file %s" % r)
elif alignments and (transcripts_gtfs or ref_gtf):
use_quant = False
if len(alignments) != n_samples:
logger.error("Aborting!")
raise Exception(
"Number of input alignment files does not match the number of samples (%s != %s)"
% (len(alignments), n_samples))
alignments = map(lambda x: x.split(","), alignments)
for i, a in enumerate(alignments):
if len(a) != n_replicates[i]:
logger.error("Aborting!")
raise Exception(
"Number of input alignment replicate files does not match the number of replicates in %d%s sample (%s != %s)"
%
(i + 1, "st" if i > 0 else "th", len(a), n_replicates[i]))
for r in a:
if not os.path.exists(r):
logger.error("Aborting!")
raise Exception("No aligment file %s" % r)
if transcripts_gtfs:
transcripts_gtfs = map(lambda x: x.split(","), transcripts_gtfs)
for i, a in enumerate(transcripts_gtfs):
if len(a) != n_replicates[i]:
logger.error("Aborting!")
raise Exception(
"Number of input gtf files does not match the total number of replicates in %d%s sample (%s != %s)"
% (i + 1, "st" if i > 0 else "th", len(a),
n_replicates[i]))
elif ref_gtf:
use_refgtf = True
if ref_gtf:
if not os.path.exists(ref_gtf):
logger.error("Aborting!")
raise Exception("No reference GTF file %s" % ref_gtf)
else:
logger.error("Aborting!")
raise Exception(
"Either (quantification files + ref_gtf) or (Alignment files + transcripts_gtfs or ref_gtf) is needed."
)
work_deseq2 = os.path.join(workdir, "deseq2", samples_txt)
create_dirs([work_deseq2])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase DESeq2 work directory for %s" % samples_txt
command = "rm -rf %s/*" % (work_deseq2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
deseq2_log = os.path.join(work_deseq2, "deseq2.log")
deseq2_log_fd = open(deseq2_log, "w")
if use_quant:
msg = "prepare tx2gene for %s." % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
tx2gene_file = ref_gtf.strip() + "tx2gene.csv"
if os.path.exists(tx2gene_file):
logger.info(
"Will use the precomputed %s as tx2gene.csv for %s" %
(tx2gene_file, samples_txt))
else:
tx2gene_file = os.path.join(work_deseq2, "tx2gene.csv")
logger.info("Will computed %s as tx2gene.csv for %s" %
(tx2gene_file, samples_txt))
tx2gene_map(ref_gtf, tx2gene_file)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "compute gene level abundances for %s." % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
fixed_quant_files = []
for i, qs in enumerate(quant_files):
fixed_qs = []
for j, q in enumerate(qs):
fixed_q = os.path.join(
work_deseq2, "{}.fixed_quant.sf".format(samples[i][j]))
fix_quant_file(q, fixed_q)
fixed_qs.append(fixed_q)
fixed_quant_files.append(fixed_qs)
command = "%s -e \"library('readr'); library('tximport'); \
samples=c(%s); (files <- file.path(c(%s))); names(files) <- samples; \
tx2gene <- read.csv(file.path('%s'),sep='\\t'); \
txi <- tximport(files, type = 'salmon', tx2gene = tx2gene); \
save(txi, file='%s/txi.rda'); \
write.table(txi$abundance, file = '%s/txi.abundances',\
quote = FALSE, sep='\\t'); \
write.table(txi$length, file = '%s/txi.length', quote = FALSE, \
sep='\\t'); write.table(txi$counts, file = '%s/txi.counts',\
quote = FALSE, sep='\\t');\"" % (
R, ",".join(
map(lambda x: "'%s'" % x,
reduce(lambda x, y: x + y, samples))), ",".join(
map(lambda x: "'%s'" % x,
reduce(lambda x, y: x + y,
fixed_quant_files))), tx2gene_file,
work_deseq2, work_deseq2, work_deseq2, work_deseq2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "DESeq2 for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s -e \"library('DESeq2'); load('%s/txi.rda'); \
samples <- c(%s); \
condition <- factor(c(%s)); \
(colData <- data.frame(row.names=colnames(txi$count), condition));\
counts <- round(txi$counts); mode(counts) <- 'integer'; \
dds <- DESeqDataSetFromMatrix(countData=counts, colData=colData, design=~ condition);\
stopifnot(txi$countsFromAbundance %%in%% c('no','scaledTPM','lengthScaledTPM')); \
if (txi$countsFromAbundance %%in%% c('scaledTPM','lengthScaledTPM')) \
{ message('using just counts from tximport'); } else \
{ message('using counts and average transcript lengths from tximport'); \
lengths <- txi$length; dimnames(lengths) <- dimnames(dds);\
assays(dds)[['avgTxLength']] <- lengths; }; \
dds <- dds[ rowSums(counts(dds)) >= %d, ]; \
dds <- DESeq(dds); \
for (i in seq_along(samples)){ \
for (j in seq_along(samples)){ \
if (i < j){\
sample1 <- samples[i]; \
sample2 <- samples[j]; \
res <- results(dds, contrast=c('condition',sample1,sample2), alpha=%f); \
(summary(res)); \
res_file= sprintf('%s/deseq2_res_%%s_vs_%%s.tab',sample1,sample2);\
write.table(res, file = res_file, \
quote = FALSE, sep='\\t'); \
} \
} \
}; \
save(txi,colData,condition,dds,res, \
file='%s/deseq2.rda');\"" % (R, work_deseq2, ",".join(
map(lambda i: "'sample%d'" %
(i), range(len(samples)))), ",".join(
map(
lambda i: "rep('sample%d', %d)" %
(i, n_replicates[i]), range(
len(samples)))), mincount, alpha, work_deseq2,
work_deseq2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
else:
if use_refgtf:
msg = "featureCounts for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------"
% step)
command = "%s %s -o %s/featureCounts.txt -T %d -a %s -g gene_id %s" % (
featureCounts, featureCounts_opts, work_deseq2, nthreads,
ref_gtf, " ".join(reduce(lambda x, y: x + y, alignments)))
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
command = "sed -i -e '2s/.*/Geneid\\tChr\\tStart\\tEnd\\tStrand\\tLength\\t%s/' %s/featureCounts.txt" % (
"\\t".join(reduce(lambda x, y: x + y,
samples)), work_deseq2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "DESeq2 for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------"
% step)
command = "%s -e \"library('DESeq2'); countData <- read.table('%s/featureCounts.txt', \
header=TRUE, row.names=1); countData <- countData[ ,6:ncol(countData)]; \
countData <- as.matrix(countData); \
samples <- c(%s); \
condition <- factor(c(%s)); \
(colData <- data.frame(row.names=colnames(countData), condition));\
dds <- DESeqDataSetFromMatrix(countData=countData, colData=colData, design=~ condition);\
dds <- dds[ rowSums(counts(dds)) >= %d, ]; \
dds <- DESeq(dds); \
for (i in seq_along(samples)){ \
for (j in seq_along(samples)){ \
if (i < j){\
sample1 <- samples[i]; \
sample2 <- samples[j]; \
res <- results(dds, contrast=c('condition',sample1,sample2), alpha=%f); \
(summary(res)); \
res_file= sprintf('%s/deseq2_res_%%s_vs_%%s.tab',sample1,sample2);\
write.table(res, file = res_file, \
quote = FALSE, sep='\\t'); \
} \
} \
}; \
save(countData,colData,condition,dds,res, \
file='%s/deseq2.rda');\"" % (
R, work_deseq2, ",".join(
map(lambda i: "'sample%d'" %
(i), range(len(samples)))), ",".join(
map(
lambda i: "rep('sample%d', %d)" %
(i, n_replicates[i]), range(
len(samples)))), mincount, alpha,
work_deseq2, work_deseq2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
else:
msg = "Merge transcripts GTFs for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------"
% step)
if ref_gtf:
stringtie_merge_opts += " -G %s" % ref_gtf
if "-p " not in stringtie_merge_opts:
stringtie_merge_opts += " -p %d" % nthreads
gtfs_list = open("%s/gtfs_list.txt" % work_deseq2, 'w')
gtfs_list.write("\n".join(
reduce(lambda x, y: x + y, transcripts_gtfs)))
gtfs_list.close()
command = "%s --merge %s -o %s/merged.gtf -v %s/gtfs_list.txt" % (
stringtie, stringtie_merge_opts, work_deseq2, work_deseq2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "featureCounts for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------"
% step)
command = "%s %s -o %s/featureCounts.txt -T %d -a %s/merged.gtf -g gene_id %s" % (
featureCounts, featureCounts_opts, work_deseq2, nthreads,
work_deseq2, " ".join(
reduce(lambda x, y: x + y, alignments)))
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
command = "sed -i -e '2s/.*/Geneid\\tChr\\tStart\\tEnd\\tStrand\\tLength\\t%s/' %s/featureCounts.txt" % (
"\\t".join(reduce(lambda x, y: x + y,
samples)), work_deseq2)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
msg = "DESeq2 for %s" % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------"
% step)
command = "%s -e \"library('DESeq2'); countData <- read.table('%s/featureCounts.txt', \
header=TRUE, row.names=1); countData <- countData[ ,6:ncol(countData)]; \
countData <- as.matrix(countData); \
samples <- c(%s); \
condition <- factor(c(%s)); \
(colData <- data.frame(row.names=colnames(countData), condition));\
dds <- DESeqDataSetFromMatrix(countData=countData, colData=colData, design=~ condition);\
dds <- dds[ rowSums(counts(dds)) >= %d, ]; \
dds <- DESeq(dds); \
for (i in seq_along(samples)){ \
for (j in seq_along(samples)){ \
if (i < j){\
sample1 <- samples[i]; \
sample2 <- samples[j]; \
res <- results(dds, contrast=c('condition',sample1,sample2), alpha=%f); \
(summary(res)); \
res_file= sprintf('%s/deseq2_res_%%s_vs_%%s.tab',sample1,sample2);\
write.table(res, file = res_file, \
quote = FALSE, sep='\\t'); \
} \
} \
}; \
save(countData,colData,condition,dds, \
file='%s/deseq2.rda');\"" % (
R, work_deseq2, ",".join(
map(lambda i: "'sample%d'" %
(i), range(len(samples)))), ",".join(
map(
lambda i: "rep('sample%d', %d)" %
(i, n_replicates[i]), range(
len(samples)))), mincount, alpha,
work_deseq2, work_deseq2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_deseq2 = os.path.join(outdir, "deseq2", samples_txt)
create_dirs([out_deseq2])
msg = "Copy predictions to output directory for %s." % samples_txt
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if len(glob.glob("%s/deseq2_res*.tab" % work_deseq2)) > 0:
for out_file in glob.glob("%s/deseq2_res*.tab" % work_deseq2):
command = "cp %s %s/" % (out_file, out_deseq2)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=deseq2_log_fd,
cmd_log=deseq2_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
diff = ""
if len(glob.glob("%s/deseq2_res*.tab" % out_deseq2)) > 0:
logger.info("DESeq2 was successfull!")
logger.info("Output differential expressions: %s" %
(glob.glob("%s/deseq2_res*.tab" % out_deseq2)))
diff = glob.glob("%s/deseq2_res*.tab" % out_deseq2)
else:
logger.info("DESeq2 failed!")
return diff
def run_salmon_smem(quantifier_idx=None,
seq_1="",
seq_2="",
seq_u="",
salmon_k=SALMON_SMEM_k,
libtype="",
salmon_smem_opts="",
salmon=SALMON,
start=0,
sample="",
nthreads=1,
unzip=False,
workdir=None,
outdir=None,
timeout=TIMEOUT):
logger.info("Running quantification (Salmon-SMEM) for %s" % sample)
if not os.path.exists(quantifier_idx):
logger.error("Aborting!")
raise Exception("No Salmon FMD index directory %s" % quantifier_idx)
if seq_1 and seq_2:
for s1 in seq_1.split(","):
if not os.path.exists(s1):
logger.error("Aborting!")
raise Exception("No Mate 1 sequence file %s" % s1)
for s2 in seq_2.split(","):
if not os.path.exists(s2):
logger.error("Aborting!")
raise Exception("No Mate 2 sequence file %s" % s2)
if unzip:
seq_argument = "-1 <(gunzip -c %s) -2 <(gunzip -c %s)" % (" ".join(
seq_1.split(",")), " ".join(seq_2.split(",")))
else:
if "," in seq_1:
seq_1 = "<(cat %s)" % (" ".join(seq_1.split(",")))
if "," in seq_2:
seq_2 = "<(cat %s)" % (" ".join(seq_2.split(",")))
seq_argument = "-1 %s -2 %s" % (seq_1, seq_2)
elif seq_u:
if unzip:
seq_argument = "-r <(gunzip -c %s)" % (" ".join(seq_u.split(",")))
elif "," in seq_u:
seq_argument = "-r <(cat %s)" % (" ".join(seq_u1.split(",")))
else:
seq_argument = "-r %s" % (seq_u)
for su in seq_u.split(","):
if not os.path.exists(su):
logger.error("Aborting!")
raise Exception("No unpaired sequence file %s" % su)
work_salmon_smem = os.path.join(workdir, "salmon_smem", sample)
create_dirs([work_salmon_smem])
step = 0
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
msg = "Erase Salmon-SMEM work directory for %s" % sample
command = "rm -rf %s/*" % (work_salmon_smem)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=False)
retcode = cmd.run(msg=msg, timeout=timeout)
step += 1
salmon_smem_log = os.path.join(work_salmon_smem, "salmon_smem.log")
salmon_smem_log_fd = open(salmon_smem_log, "w")
if "-p " not in salmon_smem_opts:
salmon_smem_opts += " -p %d" % nthreads
salmon_smem_opts += " -k %d" % salmon_k
salmon_smem_opts += " -l %s" % libtype
msg = "Salmon-SMEM for %s" % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
command = "%s quant -i %s %s %s -o %s" % (
salmon, quantifier_idx, salmon_smem_opts, seq_argument,
work_salmon_smem)
command = "bash -c \"%s\"" % command
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=salmon_smem_log_fd,
cmd_log=salmon_smem_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
out_salmon_smem = os.path.join(outdir, "salmon_smem", sample)
create_dirs([out_salmon_smem])
msg = "Copy predictions to output directory for %s." % sample
if start <= step:
logger.info(
"--------------------------STEP %s--------------------------" %
step)
if os.path.exists("%s/quant.sf" % work_salmon_smem):
command = "cp %s/quant.sf %s/quant.sf" % (work_salmon_smem,
out_salmon_smem)
cmd = TimedExternalCmd(command, logger, raise_exception=True)
retcode = cmd.run(cmd_log_fd_out=salmon_smem_log_fd,
cmd_log=salmon_smem_log,
msg=msg,
timeout=timeout)
else:
logger.info("Skipping step %d: %s" % (step, msg))
step += 1
quant = ""
if os.path.exists("%s/quant.sf" % out_salmon_smem):
logger.info("Salmon-SMEM was successfull!")
logger.info("Output expressions: %s/quant.sf" % out_salmon_smem)
quant = "%s/quant.sf" % out_salmon_smem
else:
logger.info("Salmon-SMEM failed!")
return quant
