def wrapfindcirc(files, tmp_dir, endTol, maxL, minL, strand=True, pairdendindependent=True, same=False):
# create local instance
f = Fc.Findcirc(endTol=endTol, maxL=maxL, minL=minL)
# Start de novo circular RNA detection model
sort = Fc.Sort()
indx = id_generator()
logging.info("started circRNA detection from file %s" % files)
print "started circRNA detection from file %s" % files
if same:
circfilename = files + indx + ".circRNA"
else:
circfilename = files + ".circRNA"
if pairdendindependent:
f.printcircline(files, tmp_dir + "tmp_printcirclines." + indx)
print "\t=> separating duplicates [%s]" % files
f.sepDuplicates(tmp_dir + "tmp_printcirclines." + indx, tmp_dir + "tmp_duplicates." + indx,
tmp_dir + "tmp_nonduplicates." + indx)
# Find small circles
print "\t=> locating small circRNAs [%s]" % files
f.smallcirc(tmp_dir + "tmp_duplicates." + indx, tmp_dir + "tmp_smallcircs." + indx)
if strand:
# Find normal circles
print "\t=> locating circRNAs (stranded mode) [%s]" % files
f.findcirc(tmp_dir + "tmp_nonduplicates." + indx, tmp_dir + "tmp_normalcircs." + indx, strand=True)
else:
print "\t=> locating circRNAs (unstranded mode) [%s]" % files
f.findcirc(tmp_dir + "tmp_nonduplicates." + indx, tmp_dir + "tmp_normalcircs." + indx, strand=False)
# Merge small and normal circles
print "\t=> merging circRNAs [%s]" % files
mergefiles(tmp_dir + "tmp_findcirc." + indx, tmp_dir + "tmp_smallcircs." + indx,
tmp_dir + "tmp_normalcircs." + indx)
else:
if strand:
print "\t=> locating circRNAs (stranded mode) [%s]" % files
f.findcirc(files, tmp_dir + "tmp_findcirc." + indx, strand=True)
else:
print "\t=> locating circRNAs (unstranded mode) [%s]" % files
f.findcirc(files, tmp_dir + "tmp_findcirc." + indx, strand=False)
# Sort
if strand:
print "\t=> sorting circRNAs (stranded mode) [%s]" % files
sort.sort_count(tmp_dir + "tmp_findcirc." + indx, circfilename, strand=True)
else:
print "\t=> sorting circRNAs (unstranded mode) [%s]" % files
sort.sort_count(tmp_dir + "tmp_findcirc." + indx, circfilename, strand=False)
logging.info("finished circRNA detection from file %s" % files)
print "finished circRNA detection from file %s" % files
return circfilename
def main():
version = "0.4.4"
parser = argparse.ArgumentParser(
prog="DCC",
formatter_class=argparse.RawDescriptionHelpFormatter,
fromfile_prefix_chars="@",
description=
"Contact: [email protected] || [email protected]")
parser.add_argument("--version", action="version", version=version)
parser.add_argument(
"Input",
metavar="Input",
nargs="+",
help=
"Input of the Chimeric.out.junction file from STAR. Alternatively, a sample sheet "
"specifying where your chimeric.out.junction files are, each sample per line, "
"provide with @ prefix (e.g. @samplesheet)")
parser.add_argument(
"-k",
"--keep-temp",
dest="temp",
action="store_true",
default=False,
help="Temporary files will not be deleted [default: False]")
parser.add_argument(
"-T",
"--threads",
dest="cpu_threads",
type=int,
default=2,
help="Number of CPU threads used for computation [default: 2]")
parser.add_argument("-O",
"--output",
dest="out_dir",
default="./",
help="DCC output directory [default: .]")
parser.add_argument("-t",
"--temp",
dest="tmp_dir",
default="_tmp_DCC/",
help="DCC temporary directory [default: _tmp_DCC/]")
group = parser.add_argument_group(
"Find circRNA Options", "Options to find circRNAs from STAR output")
group.add_argument(
"-D",
"--detect",
action="store_true",
dest="detect",
default=False,
help=
"Enable circRNA detection from Chimeric.out.junction files [default: False]"
)
group.add_argument(
"-ss",
action="store_true",
dest="secondstrand",
default=False,
help=
"Must be enabled for stranded libraries, aka 'fr-secondstrand' [default: False]"
)
group.add_argument("-N",
"--nonstrand",
action="store_false",
dest="strand",
default=True,
help="The library is non-stranded [default stranded]")
group.add_argument(
"-E",
"--endTol",
dest="endTol",
type=int,
default=5,
choices=range(0, 10),
help=
"Maximum base pair tolerance of reads extending over junction sites [default: 5]"
)
group.add_argument(
"-m",
"--maximum",
dest="max",
type=int,
default=1000000,
help=
"The maximum length of candidate circRNAs (including introns) [default: 1000000]"
)
group.add_argument(
"-n",
"--minimum",
dest="min",
type=int,
default=30,
help=
"The minimum length of candidate circRNAs (including introns) [default 30]"
)
group.add_argument(
"-an",
"--annotation",
dest="annotate",
help="Gene annotation file in GTF/GFF3 format, to annotate "
"circRNAs by their host gene name/identifier")
group.add_argument(
"-Pi",
"--PE-independent",
action="store_true",
dest="pairedendindependent",
default=False,
help=
"Has to be specified if the paired end mates have also been mapped separately."
"If specified, -mt1 and -mt2 must also be provided [default: False]")
group.add_argument(
"-mt1",
"--mate1",
dest="mate1",
nargs="+",
help=
"For paired end data, Chimeric.out.junction files from mate1 independent mapping result"
)
group.add_argument(
"-mt2",
"--mate2",
dest="mate2",
nargs="+",
help=
"For paired end data, Chimeric.out.junction files from mate2 independent mapping result"
)
parser.add_argument_group(group)
group = parser.add_argument_group(
"Filtering Options", "Options to filter the circRNA candidates")
group.add_argument(
"-F",
"--filter",
action="store_true",
dest="filter",
default=False,
help=
"If specified, the program will perform a recommended filter step on the detection results"
)
group.add_argument(
"-f",
"--filter-only",
dest="filteronly",
nargs=2,
help=
"If specified, the program will only filter based on two files provided: "
"1) a coordinates file [BED6 format] and 2) a count file. E.g.: -f example.bed counts.txt"
)
group.add_argument(
"-M",
"--chrM",
action="store_true",
dest="chrM",
default=False,
help=
"If specified, circRNA candidates located on the mitochondrial chromosome will be removed"
)
group.add_argument(
"-R",
"--rep_file",
dest="rep_file",
help="Custom repetitive region file in GTF format to filter out "
"circRNA candidates in repetitive regions")
group.add_argument(
"-L",
"--Ln",
dest="length",
type=int,
default=50,
help=
"Minimum length in base pairs to check for repetitive regions [default 50]"
)
group.add_argument("-Nr",
nargs=2,
type=int,
metavar=("countthreshold", "replicatethreshold"),
default=[2, 5],
help="countthreshold replicatethreshold [default: 2,5]")
group.add_argument(
"-fg",
"--filterbygene",
action="store_true",
dest="filterbygene",
default=False,
help=
"If specified, filter also by gene annotation (candidates are not allowed to span"
" more than one gene) default: False")
parser.add_argument_group(group)
group = parser.add_argument_group("Host gene count Options",
"Options to count host gene expression")
group.add_argument(
"-G",
"--gene",
action="store_true",
dest="gene",
default=False,
help=
"If specified, the program will count host gene expression given circRNA coordinates "
"[default: False]")
group.add_argument(
"-C",
"--circ",
dest="circ",
help=
"User specified circRNA coordinates, any tab delimited file with first three "
"columns as circRNA coordinates: chr\tstart\tend, which DCC will use to count "
"host gene expression")
group.add_argument(
"-B",
"--bam",
dest="bam",
nargs="+",
help=
"A file specifying the mapped BAM files from which host gene expression is computed; "
"must have the same order as input chimeric junction files")
group.add_argument("-A",
"--refseq",
dest="refseq",
help="Reference sequence FASTA file")
parser.add_argument_group(group)
options = parser.parse_args()
timestr = time.strftime("%Y-%m-%d_%H%M")
if not os.path.isdir(options.out_dir):
try:
os.makedirs(options.out_dir)
except OSError:
print "Could not create output folder %s" % options.out_dir
logging.info("Could not create output folder %s" % options.out_dir)
exit(-1)
else:
print "Output folder %s already exists, reusing" % options.out_dir
# create temporary directory if not existing
if not os.path.isdir(options.tmp_dir):
try:
os.makedirs(options.tmp_dir)
except OSError:
print "Could not create temporary folder %s" % options.tmp_dir
exit(-1)
else:
print "Temporary folder %s already exists, reusing" % options.tmp_dir
logging.basicConfig(filename=os.path.join(options.out_dir,
"DCC-" + timestr + ".log"),
filemode="w",
level=logging.DEBUG,
format="%(asctime)s %(message)s")
logging.info("DCC %s started" % version)
print "DCC %s started" % version
logging.info('DCC command line: ' + ' '.join(sys.argv))
# Get input file names
filenames = [os.path.basename(name) for name in options.Input]
samplelist = "\t".join(filenames)
# make sure the user supplied path variables have a trailing /
options.tmp_dir = os.path.join(options.tmp_dir, "")
options.out_dir = os.path.join(options.out_dir, "")
# set output file names
output_coordinates = options.out_dir + "CircCoordinates"
output_circ_counts = options.out_dir + "CircRNACount"
output_linear_counts = options.out_dir + "LinearCount"
output_skips = options.out_dir + "CircSkipJunctions"
circfiles = [] # A list for .circRNA file names
# check whether the junction file names have duplicates
same = False
if len(set(filenames)) != len(options.Input):
logging.info(
"Input file names have duplicates, add number suffix in input order to output files for distinction"
)
print(
"Input file names have duplicates, add number suffix in input order to output files for distinction"
)
same = True
cpu_count = multiprocessing.cpu_count()
if options.cpu_threads <= cpu_count:
print "%s CPU cores available, using %s" % (cpu_count,
options.cpu_threads)
else:
print "Only %s CPU cores available while %s requested, falling back to %s" % \
(cpu_count, options.cpu_threads, cpu_count)
options.cpu_threads = cpu_count
pool = multiprocessing.Pool(processes=options.cpu_threads)
if options.annotate is not None:
checkfile(options.annotate, True)
if options.rep_file is not None:
checkfile(options.rep_file, True)
# Make instance
cm = Cc.Combine(options.tmp_dir)
circann = Ca.CircAnnotate(tmp_dir=options.tmp_dir, strand=options.strand)
if (not options.mate1
or not options.mate1) and options.pairedendindependent:
logging.info(
'-Pi (Paired independent mode) specified but -mt1, -mt2, or both are missing. '
'Will not use mate information.')
print(
'-Pi (Paired independent mode) specified but -mt1, -mt2, or both are missing. '
'Will not use mate information.')
options.pairedendindependent = False
if checkjunctionfiles(options.Input, options.mate1, options.mate2,
options.pairedendindependent):
logging.info("circRNA detection skipped due to empty junction files")
print("circRNA detection skipped due to empty junction files")
options.detect = False
if options.bam and (len(options.bam) != len(options.mate1)):
logging.info(
"BAM file list is shorter than mate list. Maybe you forgot the @ (@file.list)?"
)
print(
"BAM file list is shorter than mate list. Maybe you forgot the @ (@file.list)?"
)
exit(-1)
if options.detect:
logging.info("Starting to detect circRNAs")
if options.strand:
logging.info("Stranded data mode")
else:
logging.info(
"Non-stranded data, the strand of circRNAs guessed from the strand of host genes"
)
print "WARNING: non-stranded data, the strand of circRNAs guessed from the strand of host genes"
# Start de novo circular RNA detection model
# Create instances
f = Fc.Findcirc(endTol=options.endTol,
maxL=options.max,
minL=options.min)
sort = Fc.Sort()
if options.pairedendindependent:
print "Please make sure that the read pairs have been mapped both, combined and on a per mate basis"
logging.info(
"Please make sure that the read pairs have been mapped both, combined and on a per mate basis"
)
# Fix2chimera problem by STAR
print("Collecting chimera information from mates-separate mapping")
logging.info(
"Collecting chimera information from mates-separate mapping")
Input = fixall(options.Input, options.mate1, options.mate2,
options.out_dir, options.tmp_dir)
else:
Input = options.Input
if options.strand:
if options.pairedendindependent:
circfiles = pool.map(
functools.partial(wrapfindcirc,
tmp_dir=options.tmp_dir,
endTol=options.endTol,
maxL=options.max,
minL=options.min,
strand=True,
pairdendindependent=True,
same=same), Input)
else:
circfiles = pool.map(
functools.partial(wrapfindcirc,
tmp_dir=options.tmp_dir,
endTol=options.endTol,
maxL=options.max,
minL=options.min,
strand=True,
pairdendindependent=False,
same=same), Input)
else:
if options.pairedendindependent:
circfiles = pool.map(
functools.partial(wrapfindcirc,
tmp_dir=options.tmp_dir,
endTol=options.endTol,
maxL=options.max,
minL=options.min,
strand=False,
pairdendindependent=True,
same=same), Input)
else:
circfiles = pool.map(
functools.partial(wrapfindcirc,
tmp_dir=options.tmp_dir,
endTol=options.endTol,
maxL=options.max,
minL=options.min,
strand=False,
pairdendindependent=False,
same=same), Input)
# Combine the individual count files
# Create a list of ".circRNA" file names
print("Combining individual circRNA read counts")
logging.info("Combining individual circRNA read counts")
cm.comb_coor(circfiles, strand=options.strand)
cm.map(options.tmp_dir + "tmp_coordinates",
circfiles,
strand=options.strand)
res = cm.combine([files + "mapped" for files in circfiles],
col=7,
circ=True)
# swap strand if the sequences are sense strand
if (options.secondstrand and options.strand):
logging.info("Swapping strand information")
strand_swap = {}
strand_swap["+\n"] = "-\n"
strand_swap["-\n"] = "+\n"
toswap = open(options.tmp_dir + "tmp_coordinates").readlines()
swaped = open(options.tmp_dir + "tmp_coordinatesswaped", "w")
for lin in toswap:
lin_split = lin.split("\t")
lin_split[5] = strand_swap[lin_split[5]]
swaped.write("\t".join(lin_split))
swaped.close()
os.remove(options.tmp_dir + "tmp_coordinates")
os.rename(options.tmp_dir + "tmp_coordinatesswaped",
options.tmp_dir + "tmp_coordinates")
if options.filter:
cm.writeouput(options.tmp_dir + "tmp_circCount", res)
if options.annotate:
logging.info("Write in annotation")
logging.info("Select gene features in Annotation file")
annotation_tree = circann.selectGeneGtf(options.annotate)
circann.annotate(options.tmp_dir + "tmp_coordinates",
annotation_tree,
options.tmp_dir + "tmp_coordinates_annotated")
os.remove(options.tmp_dir + "tmp_coordinates")
os.rename(options.tmp_dir + "tmp_coordinates_annotated",
options.tmp_dir + "tmp_coordinates")
else:
cm.writeouput(output_circ_counts, res, samplelist, header=True)
if options.annotate:
logging.info("Write in annotation")
logging.info("Select gene features in Annotation file")
annotation_tree = circann.selectGeneGtf(options.annotate)
circann.annotate(options.tmp_dir + "tmp_coordinates",
annotation_tree,
options.tmp_dir + "tmp_coordinates_annotated")
circann.annotateregions(
options.tmp_dir + "tmp_coordinates_annotated",
annotation_tree, output_coordinates)
else:
os.rename(options.tmp_dir + "tmp_coordinates",
output_coordinates)
# Filtering
if options.filter:
logging.info("Filtering started")
filt = Ft.Circfilter(length=options.length,
countthreshold=options.Nr[0],
replicatethreshold=options.Nr[1],
tmp_dir=options.tmp_dir)
if not options.detect and not options.gene and options.filteronly:
try:
file2filter = options.filteronly[0]
coorfile = options.filteronly[1]
logging.info("Using files %s and %s for filtering" %
(options.filteronly[0], options.filteronly[1]))
print "Using files %s and %s for filtering" % (
options.filteronly[0], options.filteronly[1])
except IndexError:
logging.error(
"Program exit because input error. Please check the input. If only use the program "
"for filtering, a coordinate file in bed6 format and a count file is needed"
)
sys.exit(
"Please check the input. If only use the program for filtering, a coordinate file in "
"bed6 format and a count file is needed")
elif not options.detect:
sys.exit(
"Filter mode for detected circRNAs enabled without detection module.\nCombine with -f or -D."
)
elif options.detect:
file2filter = options.tmp_dir + "tmp_circCount"
coorfile = options.tmp_dir + "tmp_coordinates"
logging.info(
"Using files _tmp_DCC/tmp_circCount and _tmp_DCC/tmp_coordinates for filtering"
)
print "Using files _tmp_DCC/tmp_circCount and _tmp_DCC/tmp_coordinates for filtering"
if options.rep_file:
rep_file = options.rep_file
else:
# from pkg_resources import resource_filename
# rep_file = resource_filename("DCC", "data/DCC.Repeats")
rep_file = None
count, indx = filt.readcirc(file2filter, coorfile)
logging.info("Filtering by read counts")
count0, indx0 = filt.filtercount(
count, indx) # result of first filtering by read counts
# filt.makeregion(indx0)
# nonrep_left,nonrep_right = filt.nonrep_filter("_tmp_DCC/tmp_left","_tmp_DCC/tmp_right",rep_file)
# filt.intersectLeftandRightRegions(nonrep_left,nonrep_right,indx0,count0)
if not rep_file is None:
logging.info("Filter by non repetitive region")
filt.filter_nonrep(rep_file, indx0, count0)
else:
filt.dummy_filter(indx0, count0)
if not options.chrM and not options.filterbygene:
filt.sortOutput(options.tmp_dir + "tmp_unsortedWithChrM",
output_circ_counts, output_coordinates, samplelist)
# Filter chrM, if no further filtering, return "CircRNACount" and "CircCoordinates",
# else return "_tmp_DCC/tmp_unsortedNoChrM"
if options.chrM:
logging.info(
"Deleting circRNA candidates from mitochondrial chromosome")
filt.removeChrM(options.tmp_dir + "tmp_unsortedWithChrM")
if not options.filterbygene:
filt.sortOutput(options.tmp_dir + "tmp_unsortedNoChrM",
output_circ_counts, output_coordinates,
samplelist)
else:
os.rename(options.tmp_dir + "tmp_unsortedWithChrM",
options.tmp_dir + "tmp_unsortedNoChrM")
# Note in this case "_tmp_DCC/tmp_unsortedNoChrM" actually has chrM
# Filter by gene annotation, require one circRNA could not from more than one gene.
# return final "CircRNACount" and "CircCoordinates"
if options.filterbygene:
if options.annotate:
logging.info(
"Filtering by gene annotation. "
"CircRNA candidates from more than one genes are deleted")
circann.filtbygene(
options.tmp_dir + "tmp_unsortedNoChrM",
options.tmp_dir + "tmp_unsortedfilterbygene")
filt.sortOutput(options.tmp_dir + "tmp_unsortedfilterbygene",
output_circ_counts, output_coordinates,
samplelist)
else:
logging.warning(
"To filter by gene annotation, a annotation file in GTF/GFF format needed, "
"skiped filter by gene annotation")
filt.sortOutput(options.tmp_dir + "tmp_unsortedNoChrM",
output_circ_counts, output_coordinates,
samplelist)
# Add annotation of regions
if options.annotate:
circann.annotateregions(output_coordinates, annotation_tree,
output_coordinates)
logging.info("Filtering finished")
if options.gene:
# import the list of bamfile names as a file
if not options.bam:
print "No BAM files provided (-B) trying to automatically guess BAM file names"
logging.info(
"No BAM files provided (-B) trying to automatically guess BAM file names"
)
bamfiles = convertjunctionfile2bamfile(options.Input)
if not bamfiles:
print "Could not guess BAM file names, please provides them manually via -B"
logging.info(
"Could not guess BAM file names, please provides them manually via -B"
)
else:
bamfiles = options.bam
if not options.refseq:
print "Please provide reference sequence, program will not count host gene expression"
logging.warning(
"Please provide reference sequence, program will not count host gene expression"
)
if options.refseq:
linearfiles = [] # A list for .linear file names
unsortedBAMS = checkBAMsorting(bamfiles)
# check whether the number of bamfiles is equale to the number of chimeric.junction.out files
if len(bamfiles) != len(options.Input):
logging.error(
"The number of bam files does not match with chimeric junction files"
)
sys.exit(
"The number of bam files does not match with chimeric junction files"
)
elif len(unsortedBAMS) > 0:
logging.error(
"The following BAM files seem to be not sorted by coordinate or are missing an index:"
)
logging.error(', '.join(unsortedBAMS))
print(
"The following BAM files seem to be not sorted by coordinate or are missing an index:"
)
print(', '.join(unsortedBAMS))
sys.exit(
"Error: not all BAM files are sorted by coordinate or are missing indices"
)
else:
# For each sample (each bamfile), do one host gene count, and then combine to a single table
if options.circ:
linearfiles = pool.map(
functools.partial(wraphostgenecount,
tmp_dir=options.tmp_dir,
circ_coor=options.circ,
ref=options.refseq,
countlinearsplicedreads=False),
bamfiles)
else:
linearfiles = pool.map(
functools.partial(wraphostgenecount,
tmp_dir=options.tmp_dir,
circ_coor=output_circ_counts,
ref=options.refseq,
countlinearsplicedreads=False),
bamfiles)
logging.info(
"Finished linear gene expression counting, start to combine individual sample counts"
)
# Combine all to a individual sample host gene count to a single table
res = cm.combine(linearfiles, col=6, circ=False)
cm.writeouput(output_linear_counts, res, samplelist, header=True)
logging.info(
"Finished combine individual linear gene expression counts")
if not options.temp:
deleted = cm.deletefile("", linearfiles)
logdeleted(deleted)
# CircSkip junction
if options.annotate and options.detect and not options.circ:
logging.info("Count CircSkip junctions")
print("Count CircSkip junctions")
SJ_out_tab = getSJ_out_tab(options.Input)
CircSkipfiles = findCircSkipJunction(output_coordinates,
options.tmp_dir,
options.annotate,
circfiles,
SJ_out_tab,
strand=options.strand,
same=same)
fin = open(output_coordinates, "r").readlines()[1:]
with open(options.tmp_dir + "tmp_CircCoordinatesNoheader",
"w") as fout:
fout.writelines(fin)
cm.map(options.tmp_dir + "tmp_CircCoordinatesNoheader",
CircSkipfiles,
strand=options.strand,
col=4)
CircSkipfilesmapped = [fname + "mapped" for fname in CircSkipfiles]
res = cm.combine(CircSkipfilesmapped, col=9)
cm.writeouput(output_skips, res, samplelist, header=True)
else:
logging.info(
"CircSkip junctions not counted, not running in detected mode (-D)"
)
# Delete temporary files
if not options.temp:
deletion_regex = r"^tmp_\.*"
deleted = cm.deletefile(options.tmp_dir, deletion_regex)
logdeleted(deleted)
try:
os.rmdir(options.tmp_dir)
except OSError:
print "Could not delete temporary folder %s: not empty" % options.tmp_dir
logging.info("Could not delete temporary folder %s: not empty" %
options.tmp_dir)
try:
os.rmdir(options.out_dir)
except OSError:
print "Not deleting output folder %s: contains files" % options.out_dir
logging.info("Not deleting output folder %s: contains files" %
options.out_dir)
print("Temporary files deleted")
logging.info("DCC completed successfully")
def main():
parser = argparse.ArgumentParser(
prog='DCC',
formatter_class=argparse.RawDescriptionHelpFormatter,
fromfile_prefix_chars='@',
description='Contact [email protected]')
parser.add_argument('--version',
action='version',
version='%(prog)s 0.3.2')
parser.add_argument(
"Input",
metavar='Input',
nargs="+",
help=
"Input of the chimeric.out.junction file from STAR. Alternatively, a sample sheet specifying where your chimeric.out.junction files are, each sample per line, provide with @ prefix (e.g. @samplesheet)."
)
#parser.add_argument("-O", "--output", dest="output",
# help="Tab delimited outputfile, order the same with input: \
# chr\tstart\tend\tstand\tcount\tjunctiontype")
parser.add_argument("-temp",
"--temp",
dest="temp",
action='store_true',
default=False,
help="Temporary files will not be deleted.")
group = parser.add_argument_group(
"Find circRNA Options", "Options to find circRNAs from STAR output.")
group.add_argument(
"-D",
"--detect",
action='store_true',
dest="detect",
default=False,
help=
"Always specify if you want detect circRNAs from chimeric junctions.")
group.add_argument(
"-ss",
action='store_true',
dest="secondstrand",
default=False,
help=
"For stranded libraries, specify when the library is fr-secondstrand.")
group.add_argument(
"-N",
"--nonstrand",
action='store_false',
dest="strand",
default=True,
help="Specify when the library is non-stranded [default stranded].")
group.add_argument(
"-E",
"--endTol",
dest="endTol",
type=int,
default=5,
choices=range(0, 10),
help=
"Maximum base pair tolerance of reads extending over junction sites. [Interger, default 5]"
)
group.add_argument(
"-m",
"--maximum",
dest="max",
type=int,
default=1000000,
help=
"The maximum range of candidate circRNA allowed (including introns). [default 1000000]"
)
group.add_argument(
"-n",
"--minimum",
dest="min",
type=int,
default=30,
help=
"The minimum range of candidate circRNA allowed (including introns). [default 30]"
)
group.add_argument(
"-an",
"--annotation",
dest="annotate",
help=
"Gene annotation file in GTF/GFF3 format, to annotate circRNAs by their host gene name/identifier."
)
#group.add_argument("-gf", "--getfasta", dest="getfasta",
# help="Get fasta file of circular RNAs. If a exon annotation file is provided, the circular RNA sequence will only contain annotated exons, otherwise whole sequence.")
group.add_argument(
"-Pi",
"--PE-independent",
action='store_true',
dest="pairedendindependent",
default=False,
help=
"Specify when you have mapped the PE data mates separately. If specified, -mt1 and -mt2 should also be provied. [default False]"
)
group.add_argument(
"-mt1",
"--mate1",
dest="mate1",
nargs='+',
help=
"For paired end data, Chimeric.out.juntion files from mate1 independent mapping result."
)
group.add_argument(
"-mt2",
"--mate2",
dest="mate2",
nargs='+',
help=
"For paired end data, Chimeric.out.juntion files from mate2 independent mapping result."
)
parser.add_argument_group(group)
group = parser.add_argument_group(
"Filtering Options", "Options to filter the circRNA candidates.")
group.add_argument(
"-F",
"--filter",
action='store_true',
dest="filter",
default=False,
help=
"If specified, the program will do filtering on the detection results."
)
group.add_argument(
"-M",
"--chrM",
action='store_true',
dest="chrM",
default=False,
help=
"If specified, candidates from mitochondria chromosome will be removed."
)
#group.add_argument("-J", "--junction", dest="junction",
# help="Provide a coustom junction file in gtf format, if only specify as True, only GT/AG or CT/AC junction will be considered.")
group.add_argument(
"-R",
"--rep_file",
dest="rep_file",
help=
"Custom repetitive region file in GTF format to filter out circRNAs candidates in repetitive regions."
)
group.add_argument(
"-L",
"--Ln",
dest="length",
type=int,
default=50,
help="Minimum length to check for repetitive regions. [default 50]")
group.add_argument('-Nr',
nargs=2,
type=int,
metavar=('level1', 'threshold1'),
default=[2, 5],
help='Minimum read counts required for circRNAs; \
Minimum number of samples above the corresponding expression level'
)
group.add_argument(
"-fg",
"--filterbygene",
action='store_true',
dest="filterbygene",
default=False,
help=
"If specified, filter by gene annotation. Candidates are not allowed to span more than one gene."
)
parser.add_argument_group(group)
group = parser.add_argument_group(
"Host gene count Options", "Options to count host gene expression.")
group.add_argument(
"-G",
"--gene",
action='store_true',
dest="gene",
default=False,
help=
"If specified, the program will count host gene expression given circRNA coordinates. By default, use the circRNA candidates detected from the same run."
)
group.add_argument(
"-C",
"--circ",
dest="circ",
help=
"User specified circRNA coordinates, any tab delimited file with first three columns as circRNA coordinates: chr\tstart\tend, which DCC will use to count host gene expression."
)
group.add_argument(
"-B",
"--bam",
dest="bam",
nargs='+',
help=
"A file specify the mapped bam files from which host gene expression is computed. Must have the same order as input chimeric junction files."
)
group.add_argument("-A",
"--refseq",
dest="refseq",
help="Reference sequence fasta file.")
#group.add_argument("-seq", "--seq", dest="seq",
# help="Get circRNA sequence as fasta file.")
parser.add_argument_group(group)
options = parser.parse_args()
timestr = time.strftime("%Y%m%d-%H%M%S")
logging.basicConfig(filename='DCC.log' + timestr,
filemode='w',
level=logging.DEBUG,
format='%(asctime)s %(message)s')
#root = logging.getLogger()
#root.setLevel(logging.DEBUG)
#
#ch = logging.StreamHandler(sys.stdout)
#ch.setLevel(logging.DEBUG)
#ch.filename='main.log'
#ch.filemode='w'
#formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
#ch.setFormatter(formatter)
#root.addHandler(ch)
logging.info('version:0.3.2')
logging.info(' '.join(sys.argv))
logging.info('DCC started')
try:
os.mkdir('_tmp_DCC')
except OSError:
from shutil import rmtree
rmtree('_tmp_DCC/')
os.mkdir('_tmp_DCC')
# Get input file names
filenames = [getfilename(name) for name in options.Input]
samplelist = '\t'.join(filenames)
# check whether the junction file names have duplicates
same = False
if len(set(filenames)) != len(options.Input):
logging.info(
'Input file names have duplicates, add number suffix in input order to output files for distinction.'
)
print(
'Input file names have duplicates, add number suffix in input order to output files for distinction.'
)
same = True
# Make instance
cm = CC.Combine()
circAnn = circAnnotate.CircAnnotate(strand=options.strand)
if checkjunctionfiles(options.Input, options.mate1, options.mate2):
logging.info('circRNA detection skipped due to empty junction files.')
print('circRNA detection skipped due to empty junction files.')
options.detect = False
if options.detect:
logging.info('Program start to detect circRNAs.')
if options.strand:
logging.info('Strand data.')
else:
logging.info(
'nonstrand data, the strand of circRNAs guessed from the strandness of host genes.'
)
print 'WARNING: nonstrand data, the strand of circRNAs guessed from the strandness of host genes.'
# Start de novo circular RNA detection model
# Create instances
f = FC.Findcirc(endTol=options.endTol,
maxL=options.max,
minL=options.min)
sort = FC.Sort()
circfiles = [] # A list for .circRNA file names
def wrapfindcirc(files, output, strand=True, pairdendindependent=True):
if pairdendindependent:
f.printcircline(files, '_tmp_DCC/tmp_printcirclines')
f.sepDuplicates('_tmp_DCC/tmp_printcirclines',
'_tmp_DCC/tmp_duplicates',
'_tmp_DCC/tmp_nonduplicates')
# Find small circles
f.smallcirc('_tmp_DCC/tmp_duplicates',
'_tmp_DCC/tmp_smallcircs')
if strand:
# Find normal circles
f.findcirc('_tmp_DCC/tmp_nonduplicates',
'_tmp_DCC/tmp_normalcircs',
strand=True)
else:
f.findcirc('_tmp_DCC/tmp_nonduplicates',
'_tmp_DCC/tmp_normalcircs',
strand=False)
# Merge small and normal circles
mergefiles('_tmp_DCC/tmp_findcirc', '_tmp_DCC/tmp_smallcircs',
'_tmp_DCC/tmp_normalcircs')
else:
if strand:
f.findcirc(files, '_tmp_DCC/tmp_findcirc', strand=True)
else:
f.findcirc(files, '_tmp_DCC/tmp_findcirc', strand=False)
# Sort
if strand:
sort.sort_count('_tmp_DCC/tmp_findcirc', output, strand=True)
else:
sort.sort_count('_tmp_DCC/tmp_findcirc', output, strand=False)
if options.pairedendindependent:
print 'Please make sure that the read pairs have been mapped both, combined and on a per mate basis'
logging.info(
"Please make sure that the read pairs have been mapped both, combined and on a per mate basis"
)
# Fix2chimera problem by STAR
print('Collecting chimera from mates-separate mapping.')
logging.info('Collecting chimera from mates-separate mapping.')
Input = fixall(options.Input, options.mate1, options.mate2)
else:
Input = options.Input
for indx, files in enumerate(Input):
logging.info('started circRNA detection based on %s' % files)
print 'started circRNA detection based on %s' % files
if same:
circfilename = '_tmp_DCC/' + getfilename(files) + str(
indx) + '.circRNA'
else:
circfilename = '_tmp_DCC/' + getfilename(files) + '.circRNA'
circfiles.append(circfilename)
if options.strand:
if options.pairedendindependent:
wrapfindcirc(files,
circfilename,
strand=True,
pairdendindependent=True)
else:
wrapfindcirc(files,
circfilename,
strand=True,
pairdendindependent=False)
else:
if options.pairedendindependent:
wrapfindcirc(files,
circfilename,
strand=False,
pairdendindependent=True)
else:
wrapfindcirc(files,
circfilename,
strand=False,
pairdendindependent=False)
#
#try:
# os.remove('_tmp_DCC/tmp_findcirc')
# os.remove('_tmp_DCC/tmp_printcirclines')
# os.remove('_tmp_DCC/tmp_duplicates')
# os.remove('_tmp_DCC/tmp_nonduplicates')
# os.remove('_tmp_DCC/tmp_smallcircs')
#except OSError:
# pass
### Combine the individual count files
# Create a list of '.circRNA' file names
print('Combining individual circRNA read counts.')
logging.info('Combining individual circRNA read counts.')
cm.comb_coor(circfiles, strand=options.strand)
cm.map('_tmp_DCC/tmp_coordinates', circfiles, strand=options.strand)
res = cm.combine([files + 'mapped' for files in circfiles],
col=7,
circ=True)
# swap strand if the sequences are sense strand
if (options.secondstrand and options.strand):
logging.info('Swapping strand information.')
strand_swap = {}
strand_swap['+\n'] = '-\n'
strand_swap['-\n'] = '+\n'
toswap = open('_tmp_DCC/tmp_coordinates').readlines()
swaped = open('_tmp_DCC/tmp_coordinatesswaped', 'w')
for lin in toswap:
lin_split = lin.split('\t')
lin_split[5] = strand_swap[lin_split[5]]
swaped.write('\t'.join(lin_split))
swaped.close()
os.remove('_tmp_DCC/tmp_coordinates')
os.rename('_tmp_DCC/tmp_coordinatesswaped',
'_tmp_DCC/tmp_coordinates')
if options.filter:
cm.writeouput('_tmp_DCC/tmp_circCount', res)
if options.annotate:
logging.info('Writing annotation.')
logging.info('Selecting gene features in Annotation file.')
circAnn.selectGeneGtf(options.annotate)
circAnn.annotate(
'_tmp_DCC/tmp_coordinates',
'_tmp_DCC/tmp_' + getfilename(options.annotate) + '.gene',
'_tmp_DCC/tmp_coordinatesannotated')
os.remove('_tmp_DCC/tmp_coordinates')
os.rename('_tmp_DCC/tmp_coordinatesannotated',
'_tmp_DCC/tmp_coordinates')
else:
cm.writeouput('CircRNACount', res, samplelist, header=True)
if options.annotate:
logging.info('Write in annotation.')
logging.info('Select gene features in Annotation file.')
circAnn.selectGeneGtf(options.annotate)
circAnn.annotate(
'_tmp_DCC/tmp_coordinates',
'_tmp_DCC/tmp_' + getfilename(options.annotate) + '.gene',
'CircCoordinates')
circAnn.annotateregions('CircCoordinates', options.annotate)
else:
os.rename('_tmp_DCC/tmp_coordinates', 'CircCoordinates')
### Filtering
if options.filter:
logging.info('Filtering started')
import circFilter as FT
filt = FT.Circfilter(length=options.length,
level1=options.Nr[0],
threshold1=options.Nr[1])
if not options.detect and len(
options.Input
) == 2: # Only use the program for filtering, need one coordinates file (bed6), one count file
try:
file2filter = options.Input[0]
coorfile = options.Input[1]
logging.info('Using files %s and %s for filtering' %
(options.Input[0], options.Input[1]))
print 'Using files %s and %s for filtering' % (
options.Input[0], options.Input[1])
except IndexError:
sys.exit(
'Please check the input. If only use the program for filtering, a coordinate file in bed6 format and a count file is needed.'
)
logging.error(
'Program exit because input error. Please check the input. If only use the program for filtering, a coordinate file in bed6 format and a count file is needed.'
)
elif options.detect:
file2filter = '_tmp_DCC/tmp_circCount'
coorfile = '_tmp_DCC/tmp_coordinates'
logging.info(
'Take file _tmp_DCC/tmp_circCount and _tmp_DCC/tmp_coordinates for filtering'
)
print 'Using files _tmp_DCC/tmp_circCount and _tmp_DCC/tmp_coordinates for filtering'
if options.rep_file:
rep_file = options.rep_file
else:
from pkg_resources import resource_filename
rep_file = resource_filename('DCC', 'data/DCC.Repeats')
count, indx = filt.readcirc(file2filter, coorfile)
logging.info('Filtering by read counts.')
count0, indx0 = filt.filtercount(
count, indx) # result of first filtering by read counts
filt.makeregion(indx0)
logging.info('Filtering by non repetitive regions.')
nonrep_left, nonrep_right = filt.nonrep_filter('_tmp_DCC/tmp_left',
'_tmp_DCC/tmp_right',
rep_file)
filt.intersectLeftandRightRegions(nonrep_left, nonrep_right, indx0,
count0)
if not options.chrM and not options.filterbygene:
filt.sortOutput('_tmp_DCC/tmp_unsortedWithChrM',
'CircRNACount',
samplelist,
'CircCoordinates',
split=True)
# Filter chrM, if no further filtering, return 'CircRNACount' and 'CircCoordinates', else return '_tmp_DCC/tmp_unsortedNoChrM'
if options.chrM:
logging.info(
'Deleting circRNA candidates from mitochondrial chromosome.')
filt.removeChrM('_tmp_DCC/tmp_unsortedWithChrM')
if not options.filterbygene:
filt.sortOutput('_tmp_DCC/tmp_unsortedNoChrM',
'CircRNACount',
samplelist,
'CircCoordinates',
split=True)
else:
os.rename(
'_tmp_DCC/tmp_unsortedWithChrM', '_tmp_DCC/tmp_unsortedNoChrM'
) # Note in this case '_tmp_DCC/tmp_unsortedNoChrM' actually has chrM
# Filter by gene annotation, require one circRNA could not from more than one gene. return final 'CircRNACount' and 'CircCoordinates'
if options.filterbygene:
if options.annotate:
logging.info(
'Filtering by gene annotation. CircRNA candidates from more than one genes are deleted.'
)
circAnn.filtbygene('_tmp_DCC/tmp_unsortedNoChrM',
'_tmp_DCC/tmp_unsortedfilterbygene')
filt.sortOutput('_tmp_DCC/tmp_unsortedfilterbygene',
'CircRNACount',
samplelist,
'CircCoordinates',
split=True)
else:
logging.warning(
'To filter by gene annotation, a annotation file in GTF/GFF format needed, skiped filter by gene annotation.'
)
filt.sortOutput('_tmp_DCC/tmp_unsortedNoChrM',
'CircRNACount',
samplelist,
'CircCoordinates',
split=True)
# Add annotation of regions
if options.annotate:
circAnn.annotateregions('CircCoordinates', options.annotate)
logging.info('Filtering finished')
if options.gene:
import genecount as GC
# import the list of bamfile names as a file
if not options.bam:
#print 'Please provide bam files, program will not count host gene expression.'
logging.info(
'Look for mapped bam files in the same directory as chimeric.out.junction files.'
)
bamfiles = convertjunctionfile2bamfile(options.Input)
else:
bamfiles = options.bam
if not options.refseq:
print 'Please provide reference sequence, program will not count host gene expression.'
logging.warning(
'Please provide reference sequence, program will not count host gene expression.'
)
if options.refseq:
# check whether the number of bamfiles is equale to the number of chimeric.junction.out files
if len(bamfiles) != len(options.Input):
logging.error(
"The number of bam files does not match with chimeric junction files."
)
sys.exit(
"The number of bam files does not match with chimeric junction files."
)
else:
# For each sample (each bamfile), do one host gene count, and then combine to a single table
gc = GC.Genecount()
linearfiles = [] # A list for .linear file names
for indx, files in enumerate(options.Input):
if same:
linearfilename = '_tmp_DCC/' + getfilename(
files) + str(indx) + '.linear'
else:
linearfilename = '_tmp_DCC/' + getfilename(
files) + '.linear'
linearfiles.append(linearfilename)
for indx, bamfile in enumerate(bamfiles):
if options.circ:
logging.info(
'Counting linear gene expression based on provided circRNA coordinates for %s'
% bamfile)
#print 'Counting linear gene expression based on provided circRNA coordinates'
gc.comb_gen_count(options.circ,
bamfile,
options.refseq,
linearfiles[indx],
countlinearsplicedreads=False)
else:
logging.info(
'Counting host gene expression based on detected and filtered circRNA coordinates for %s'
% bamfile)
print 'Counting host gene expression based on detected and filtered circRNA coordinates'
gc.comb_gen_count('CircRNACount',
bamfile,
options.refseq,
linearfiles[indx],
countlinearsplicedreads=False)
logging.info(
"Finished linear gene expression counting, start to combine individual sample counts"
)
# Combine all to a individual sample host gene count to a single table
res = cm.combine(linearfiles, col=6, circ=False)
cm.writeouput('LinearCount', res, samplelist, header=True)
logging.info(
'Finished combine individual linear gene expression counts')
if not options.temp:
deleted = cm.deletfile(os.getcwd(), linearfiles)
logdeleted(deleted)
# CircSkip junction
if options.annotate and not options.circ:
logging.info('Count CircSkip junctions.')
print('Count CircSkip junctions.')
SJ_out_tab = getSJ_out_tab(options.Input)
CircSkipfiles = findCircSkipJunction('CircCoordinates',
options.annotate,
circfiles,
SJ_out_tab,
strand=options.strand,
same=same)
fin = open('CircCoordinates', 'r').readlines()[1:]
with open('_tmp_DCC/tmp_CircCoordinatesNoheader', 'w') as fout:
fout.writelines(fin)
cm.map('_tmp_DCC/tmp_CircCoordinatesNoheader',
CircSkipfiles,
strand=options.strand,
col=4)
CircSkipfilesmapped = [fname + 'mapped' for fname in CircSkipfiles]
res = cm.combine(CircSkipfilesmapped, col=9)
cm.writeouput('CircSkipJunctions', res, samplelist, header=True)
else:
logging.info('CircSkip junctions cannot be count.')
# Delete temporary files
if not options.temp:
p3 = r'^tmp_\.*'
deleted = cm.deletfile(os.path.join(os.getcwd(), '_tmp_DCC/'), p3)
logdeleted(deleted)
deleted = cm.deletfile(
os.getcwd(), circfiles + [files + 'mapped' for files in circfiles])
logdeleted(deleted)
deleted = cm.deletfile('', CircSkipfiles)
logdeleted(deleted)
deleted = cm.deletfile('', CircSkipfilesmapped)
logdeleted(deleted)
logging.info('DCC completed successfully.')