def picard_collectInsertSizeMetrics(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get("PICARD_HOME"): picard = os.environ.get("PICARD_HOME") elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] histWide = library * 2 command = [ "java", "-Xmx16g", "-XX:PermSize=2g", "-jar", os.path.join(picard, "CollectInsertSizeMetrics.jar"), "INPUT={}".format(BAMfile), "MINIMUM_PCT=0", "HISTOGRAM_FILE={}.collectInsertSize.pdf".format(output_header), "OUTPUT={}.collectInsertSize.txt".format(output_header), "HISTOGRAM_WIDTH={}".format(histWide), "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch", ] returnValue = 0 if not os.path.exists("{}.collectInsertSize.pdf".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("collectInsertSize.stdOut", "w") stdErr = open("collectInsertSize.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running CollectInsertSizeMetrics" os.chdir("..") return sorted_alignments_by_insert
def picard_markDuplicates(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get("PICARD_HOME"): picard = os.environ.get("PICARD_HOME") elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] command = [ "java", "-Xmx16g", "-XX:PermSize=3g", "-jar", os.path.join(picard, "MarkDuplicates.jar"), "INPUT={}".format(BAMfile), "OUTPUT={}_noDup.bam".format(output_header), "METRICS_FILE={0}.markDuplicates.txt".format(output_header), "ASSUME_SORTED=true", "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch", ] returnValue = 0 if not os.path.exists("{}_noDup.bam".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("removeDup.stdOut", "w") stdErr = open("removeDup.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running MarkDuplicates" os.chdir("..") return sorted_alignments_by_insert
def picard_markDuplicates(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get('PICARD_HOME'): picard = os.environ.get('PICARD_HOME') elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] command= ["java", "-Xmx16g", "-XX:PermSize=3g", "-jar", os.path.join(picard, "MarkDuplicates.jar"), "INPUT={}".format(BAMfile), "OUTPUT={}_noDup.bam".format(output_header), \ "METRICS_FILE={0}.markDuplicates.txt".format(output_header), "ASSUME_SORTED=true", "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch"] returnValue = 0 if not os.path.exists("{}_noDup.bam".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("removeDup.stdOut", "w") stdErr = open("removeDup.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running MarkDuplicates" os.chdir("..") return sorted_alignments_by_insert
def picard_collectInsertSizeMetrics(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get('PICARD_HOME'): picard = os.environ.get('PICARD_HOME') elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] histWide = library * 2 command = [ "java", "-Xmx16g", "-XX:PermSize=2g", "-jar", os.path.join(picard, "CollectInsertSizeMetrics.jar"), "INPUT={}".format(BAMfile), "MINIMUM_PCT=0", "HISTOGRAM_FILE={}.collectInsertSize.pdf".format(output_header), "OUTPUT={}.collectInsertSize.txt".format(output_header), "HISTOGRAM_WIDTH={}".format(histWide), "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch" ] returnValue = 0 if not os.path.exists( "{}.collectInsertSize.pdf".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("collectInsertSize.stdOut", "w") stdErr = open("collectInsertSize.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running CollectInsertSizeMetrics" os.chdir("..") return sorted_alignments_by_insert
def picard_CGbias(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get("PICARD_HOME"): picard = os.environ.get("PICARD_HOME") elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] command = [ "java", "-Xmx16g", "-XX:PermSize=2g", "-jar", os.path.join(picard, "CollectGcBiasMetrics.jar"), "REFERENCE_SEQUENCE={}".format(sample_config["reference"]), "INPUT={}".format(BAMfile), "OUTPUT={}.collectGcBias.txt".format(output_header), "CHART_OUTPUT={}.collectGcBias.pdf".format(output_header), "ASSUME_SORTED=true", "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch", ] returnValue = 0 if not os.path.exists("{}.collectGcBias.pdf".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("collectGcBias.stdOut", "w") stdErr = open("collectGcBias.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running collectGCBias" os.chdir("..") return sorted_alignments_by_insert
def align_bwa_mem(global_config, read1, read2, reference, threads, dryrun): aligner = "bwa" if "bwa" in global_config["Tools"]: aligner = global_config["Tools"]["bwa"]["bin"] elif not common.which("bwa"): sys.exit("error while trying to run bwa mem: bwa not present in the path and not in global config, please make sure to install bwa properly") samtools = "samtools" if "samtools" in global_config["Tools"]: samtools = global_config["Tools"]["samtools"]["bin"] elif not common.which("samtools"): sys.exit("error while trying to run samtools: bwa not present in the path and not in global config, please make sure to install bwa properly") # extract base name libraryBase = "" if read2: libraryBase = os.path.basename(read1).split("_1.fastq")[0] else: libraryBase = os.path.basename(read1).split(".fastq")[0] if not os.path.exists(libraryBase): os.makedirs(libraryBase) os.chdir(libraryBase) mappingBase = "{}_to_{}".format(libraryBase, os.path.basename(reference).split(".fasta")[0]) BAMsorted = "{}.bam".format(mappingBase) BAMunsorted = "{}.unsorted.bam".format(mappingBase) SAMMapped = "{}.unsorted.sam".format(mappingBase) if os.path.exists(os.path.abspath(BAMsorted)): BAMsorted = os.path.abspath(BAMsorted) os.chdir("..") return BAMsorted bwa_mem_command = [aligner, "mem", "-M", "-t", "{}".format(threads), reference, read1, read2] samtools_view_command = [samtools, "view", "-b", "-S", "-u", "-"] if not os.path.exists(BAMunsorted): command = "{} | {} > {}".format(" ".join(bwa_mem_command), " ".join(samtools_view_command), BAMunsorted) bwa_stdOut = open("bwa.stdOut", "w") bwa_stdErr = open("bwa.stdErr", "w") common.print_command(command) if not dryrun: subprocess.call(command, shell=True, stdout=bwa_stdOut, stderr=bwa_stdErr) samtools_sort_command = [samtools, "sort", "-@", "{}".format(threads), "-m" , "1G", BAMunsorted, mappingBase] command = " ".join(samtools_sort_command) if not os.path.exists(BAMsorted): stdOut = open("sam_sort.stdOut", "w") stdErr = open("sam_sort.stdErr", "w") common.print_command(command) if not dryrun: subprocess.call(command, shell=True, stdout=stdOut, stderr=stdErr) if os.path.exists(BAMsorted) and os.path.exists(BAMunsorted) : subprocess.call(["rm", BAMunsorted]) BAMsorted = os.path.abspath(BAMsorted) os.chdir("..") return BAMsorted
def build_reference_bwa(global_config, sample_config): #build the reference if not available reference = sample_config["reference"] program = "bwa" if "bwa" in global_config["Tools"]: program = global_config["Tools"]["bwa"]["bin"] elif not common.which("bwa"): sys.exit( "error while trying to run bwa index: bwa not present in the path and not in global config, please make sure to install bwa properly" ) # check if reference provided exisists reference = os.path.abspath(reference) if not os.path.exists(reference): sys.exit("error, reference file {} does not exists".format(reference)) # check if bwa index already created current_dir = os.getcwd() path_name, base_name = os.path.split(reference) bwa_index_folder = os.path.join(path_name, "bwa") #if needed create directory if not os.path.exists(bwa_index_folder): os.makedirs(bwa_index_folder) os.chdir(bwa_index_folder) # if needed soft link the reference if not os.path.exists(base_name): returnValue = subprocess.call(["ln", "-s", reference, base_name]) if not returnValue == 0: sys.exit("error while trying to soft link reference sequence") reference = os.path.join(path_name, "bwa", base_name) # now I have a soflinked copy # now check if index alredy build or not if not os.path.exists( "{}.bwt".format(reference)): # then create the index sequence bwa_stdOut = open("bwa_index.stdOut", "w") bwa_stdErr = open("bwa_index.stdErr", "w") command = [program, "index", reference] common.print_command(command) if not common.check_dryrun(sample_config): returnValue = subprocess.call(command, stdout=bwa_stdOut, stderr=bwa_stdErr) if not returnValue == 0: sys.exit( "error, while indexing reference file {} with bwa index". format(reference)) #extra control to avoid problem with unexpected return value if not os.path.exists("{}.bwt".format(reference)): sys.exit("bwa index failed") os.chdir(current_dir) return reference
def build_reference_bwa(global_config, sample_config): # build the reference if not available reference = sample_config["reference"] program = "bwa" if "bwa" in global_config["Tools"]: program = global_config["Tools"]["bwa"]["bin"] elif not common.which("bwa"): sys.exit( "error while trying to run bwa index: bwa not present in the path and not in global config, please make sure to install bwa properly" ) # check if reference provided exisists reference = os.path.abspath(reference) # check if I have already the bwt index if os.path.exists("{}.bwt".format(reference)): # index already present, nothing to do return reference # otherwise I need to build the reference, in this case I build it locally if not os.path.exists(reference): sys.exit("error, reference file {} does not exists".format(reference)) # check if bwa index already created current_dir = os.getcwd() path_name, base_name = os.path.split(reference) bwa_index_folder = os.path.join(path_name, "bwa") # if needed create directory if not os.path.exists(bwa_index_folder): os.makedirs(bwa_index_folder) os.chdir(bwa_index_folder) # if needed soft link the reference if not os.path.exists(base_name): returnValue = subprocess.call(["ln", "-s", reference, base_name]) if not returnValue == 0: sys.exit("error while trying to soft link reference sequence") reference = os.path.join(path_name, "bwa", base_name) # now I have a soflinked copy # now check if index alredy build or not if not os.path.exists("{}.bwt".format(reference)): # then create the index sequence bwa_stdOut = open("bwa_index.stdOut", "w") bwa_stdErr = open("bwa_index.stdErr", "w") command = [program, "index", reference] common.print_command(command) if not common.check_dryrun(sample_config): returnValue = subprocess.call(command, stdout=bwa_stdOut, stderr=bwa_stdErr) if not returnValue == 0: sys.exit("error, while indexing reference file {} with bwa index".format(reference)) # extra control to avoid problem with unexpected return value if not os.path.exists("{}.bwt".format(reference)): sys.exit("bwa index failed") os.chdir(current_dir) return reference
def picard_CGbias(global_config, sample_config, sorted_alignments_by_insert): picard = "" if os.environ.get('PICARD_HOME'): picard = os.environ.get('PICARD_HOME') elif "picard" in global_config["Tools"]: picard = global_config["Tools"]["picard"]["bin"] for library, BAMfile, working_dir in sorted_alignments_by_insert: os.chdir(working_dir) output_header = os.path.basename(BAMfile).split(".bam")[0] command= ["java", "-Xmx16g", "-XX:PermSize=2g", "-jar", os.path.join(picard, "CollectGcBiasMetrics.jar"), "REFERENCE_SEQUENCE={}".format(sample_config["reference"]), "INPUT={}".format(BAMfile), \ "OUTPUT={}.collectGcBias.txt".format(output_header), "CHART_OUTPUT={}.collectGcBias.pdf".format(output_header), "ASSUME_SORTED=true", "VALIDATION_STRINGENCY=LENIENT", "TMP_DIR=/scratch"] returnValue = 0 if not os.path.exists("{}.collectGcBias.pdf".format(output_header)): common.print_command(command) if not common.check_dryrun(sample_config): stdOut = open("collectGcBias.stdOut", "w") stdErr = open("collectGcBias.stdErr", "w") returnValue = subprocess.call(command, stdout=stdOut, stderr=stdErr) if not returnValue == 0: print "problem running collectGCBias" os.chdir("..") return sorted_alignments_by_insert
def _merge_bam_files(global_config, sample_config, sorted_libraries_by_insert): BAMfiles = {} reference = sample_config["reference"] samtools = "samtools" if "samtools" in global_config["Tools"]: samtools = global_config["Tools"]["samtools"]["bin"] elif not common.which("samtools"): sys.exit( "error while trying to run samtools: bwa not present in the path and not in global config, please make sure to install bwa properly" ) numInserts = 0 for library, libraryInfo in sorted_libraries_by_insert: read1 = libraryInfo["pair1"] read2 = libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] alignment = libraryInfo["alignment"] if insert not in BAMfiles: BAMfiles[insert] = [alignment] numInserts += 1 else: BAMfiles[insert].append(alignment) BAMfilesMerged = {} for insert, insertGroup in BAMfiles.iteritems(): dir_insert = "lib_{}".format(insert) if numInserts == 1: dir_insert = sample_config["output"] if not os.path.exists(dir_insert): os.makedirs(dir_insert) os.chdir(dir_insert) # check if file is already present bamMerged = "lib_{}.bam".format(insert) if numInserts == 1: bamMerged = "{}.bam".format(sample_config["output"]) if os.path.exists(bamMerged): BAMfilesMerged[insert] = [os.path.abspath(bamMerged), dir_insert] os.chdir("..") continue # nothiing to be done for this insert if len(insertGroup) == 1: # only one sample file for this insert length cl = ["ln", "-s", insertGroup[0], bamMerged] returnValue = subprocess.call(cl) if not returnValue == 0: sys.exit("error, while soft linking {}".format(insertGroup[0])) else: command = [samtools, "merge", bamMerged] for bamfile in insertGroup: command.append(bamfile) common.print_command(command) returnValue = 0 if not common.check_dryrun(sample_config): returnValue = subprocess.call(command) if not returnValue == 0: sys.exit("error, while merging files {}".format(insertGroup)) BAMfilesMerged[insert] = [os.path.abspath(bamMerged), dir_insert] os.chdir("..") sorted_alignments_by_insert = [] for key in sorted(BAMfilesMerged.iterkeys()): sorted_alignments_by_insert.append( [key, BAMfilesMerged[key][0], BAMfilesMerged[key][1]] ) # memorise insert length, bam file, folder return sorted_alignments_by_insert
def _run_cabog(global_config, sample_config, sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "cabog" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in cabog case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure(assembler, assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config ########### HERE IT START THE SPECIFIC ASSEMBLER PART sys.path.insert(0, programBIN) libraries = 1 for library, libraryInfo in sorted_libraries_by_insert: command_fastqToCA = os.path.join(programBIN, "fastqToCA") read1=libraryInfo["pair1"] read2=libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] command_fastqToCA += " -libraryname " command_fastqToCA += " {}_{}".format(outputName, libraries) command_fastqToCA += " -insertsize " command_fastqToCA += " {} {} ".format(insert,std) command_fastqToCA += " -technology " command_fastqToCA += " illumina " command_fastqToCA += " -type " command_fastqToCA += " illumina " if orientation=="innie" or orientation=="none" : command_fastqToCA += " -innie " if read2 is None: command_fastqToCA += " -reads " command_fastqToCA += " {} ".format(read1) else: command_fastqToCA += " -mates " command_fastqToCA += " {},{} ".format(read1, read2) elif orientation=="outtie": command_fastqToCA += " -outtie " command_fastqToCA += " -mates " command_fastqToCA += " {},{} ".format(read1, read2) command_fastqToCA += " > " command_fastqToCA += " {}_{}.frg ".format(outputName, libraries) common.print_command(command_fastqToCA) if not common.check_dryrun(sample_config): cabog_stdOut = open("cabog_fastqToCA.stdOut", "w") cabog_stdErr = open("cabogfastqToCA.stdErr", "w") subprocess.call(command_fastqToCA, stderr=cabog_stdErr, shell=True) cabog_stdOut.close() cabog_stdErr.close() libraries += 1 command_runCA = os.path.join(programBIN, "runCA") command_runCA += " -d runCABOGfolder -p {} *frg".format(outputName) common.print_command(command_runCA) if common.check_dryrun(sample_config): return sample_config returnValue = 0 cabog_stdOut = open("cabog_runCA.stdOut", "w") cabog_stdErr = open("cabog_runCA.stdErr", "w") returnValue = subprocess.call(command_runCA, stdout=cabog_stdOut, stderr=cabog_stdErr, shell=True) if returnValue == 0: #assembly succed, remove files and save assembly if os.path.exists(os.path.join("runCABOGfolder","9-terminator", "{}.ctg.fasta".format(outputName))): subprocess.call(["mv", os.path.join("runCABOGfolder","9-terminator", "{}.ctg.fasta".format(outputName)), "{}.ctg.fasta".format(outputName)]) subprocess.call(["mv", os.path.join("runCABOGfolder","9-terminator", "{}.scf.fasta".format(outputName)), "{}.scf.fasta".format(outputName)]) subprocess.call(["rm", "-r", "runCABOGfolder"]) else: print "something wrong with CABOG -> no contig file generated" else: print "CABOG terminated with an error. Please check running folder for more informations" os.chdir("..") return sample_config
def _merge_bam_files(global_config, sample_config, sorted_libraries_by_insert): BAMfiles = {} reference = sample_config["reference"] samtools = "samtools" if "samtools" in global_config["Tools"]: samtools = global_config["Tools"]["samtools"]["bin"] elif not common.which("samtools"): sys.exit( "error while trying to run samtools: bwa not present in the path and not in global config, please make sure to install bwa properly" ) numInserts = 0 for library, libraryInfo in sorted_libraries_by_insert: read1 = libraryInfo["pair1"] read2 = libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] alignment = libraryInfo["alignment"] if insert not in BAMfiles: BAMfiles[insert] = [alignment] numInserts += 1 else: BAMfiles[insert].append(alignment) BAMfilesMerged = {} for insert, insertGroup in BAMfiles.iteritems(): dir_insert = "lib_{}".format(insert) if numInserts == 1: dir_insert = sample_config["output"] if not os.path.exists(dir_insert): os.makedirs(dir_insert) os.chdir(dir_insert) #check if file is already present bamMerged = "lib_{}.bam".format(insert) if numInserts == 1: bamMerged = "{}.bam".format(sample_config["output"]) if os.path.exists(bamMerged): BAMfilesMerged[insert] = [os.path.abspath(bamMerged), dir_insert] os.chdir("..") continue # nothiing to be done for this insert if len(insertGroup ) == 1: # only one sample file for this insert length cl = ["ln", "-s", insertGroup[0], bamMerged] returnValue = subprocess.call(cl) if not returnValue == 0: sys.exit("error, while soft linking {}".format(insertGroup[0])) else: command = [samtools, "merge", bamMerged] for bamfile in insertGroup: command.append(bamfile) common.print_command(command) returnValue = 0 if not common.check_dryrun(sample_config): returnValue = subprocess.call(command) if not returnValue == 0: sys.exit( "error, while merging files {}".format(insertGroup)) BAMfilesMerged[insert] = [os.path.abspath(bamMerged), dir_insert] os.chdir("..") sorted_alignments_by_insert = [] for key in sorted(BAMfilesMerged.iterkeys()): sorted_alignments_by_insert.append([ key, BAMfilesMerged[key][0], BAMfilesMerged[key][1] ]) # memorise insert length, bam file, folder return sorted_alignments_by_insert
def align_bwa_mem(global_config, read1, read2, reference, threads, dryrun): aligner = "bwa" if "bwa" in global_config["Tools"]: aligner = global_config["Tools"]["bwa"]["bin"] elif not common.which("bwa"): sys.exit( "error while trying to run bwa mem: bwa not present in the path and not in global config, please make sure to install bwa properly" ) samtools = "samtools" if "samtools" in global_config["Tools"]: samtools = global_config["Tools"]["samtools"]["bin"] elif not common.which("samtools"): sys.exit( "error while trying to run samtools: bwa not present in the path and not in global config, please make sure to install bwa properly" ) # extract base name libraryBase = "" if read2: libraryBase = os.path.basename(read1).split("_1.fastq")[0] else: libraryBase = os.path.basename(read1).split(".fastq")[0] if not os.path.exists(libraryBase): os.makedirs(libraryBase) os.chdir(libraryBase) mappingBase = "{}_to_{}".format( libraryBase, os.path.basename(reference).split(".fasta")[0]) BAMsorted = "{}.bam".format(mappingBase) BAMunsorted = "{}.unsorted.bam".format(mappingBase) SAMMapped = "{}.unsorted.sam".format(mappingBase) if os.path.exists(os.path.abspath(BAMsorted)): BAMsorted = os.path.abspath(BAMsorted) os.chdir("..") return BAMsorted bwa_mem_command = [ aligner, "mem", "-M", "-t", "{}".format(threads), reference, read1, read2 ] samtools_view_command = [samtools, "view", "-b", "-S", "-u", "-"] if not os.path.exists(BAMunsorted): command = "{} | {} > {}".format(" ".join(bwa_mem_command), " ".join(samtools_view_command), BAMunsorted) bwa_stdOut = open("bwa.stdOut", "w") bwa_stdErr = open("bwa.stdErr", "w") common.print_command(command) if not dryrun: subprocess.call(command, shell=True, stdout=bwa_stdOut, stderr=bwa_stdErr) samtools_sort_command = [ samtools, "sort", "-@", "{}".format(threads), "-m", "1G", BAMunsorted, mappingBase ] command = " ".join(samtools_sort_command) if not os.path.exists(BAMsorted): stdOut = open("sam_sort.stdOut", "w") stdErr = open("sam_sort.stdErr", "w") common.print_command(command) if not dryrun: subprocess.call(command, shell=True, stdout=stdOut, stderr=stdErr) if os.path.exists(BAMsorted) and os.path.exists(BAMunsorted): subprocess.call(["rm", BAMunsorted]) BAMsorted = os.path.abspath(BAMsorted) os.chdir("..") return BAMsorted
def _run_allpaths(global_config, sample_config, sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "allpaths" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in abyss case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure("abyss", assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config inGroups_file = open("in_groups.csv", "w") inLibs_file = open("in_libs.csv", "w") inGroups_file.write("group_name, library_name, file_name\n") inLibs_file.write("library_name, project_name, organism_name, type, paired, frag_size, frag_stddev, insert_size, insert_stddev, read_orientation,genomic_start, genomic_end\n") librariesForInLibs = [] librariesForInLibsDict = {} group_name = 1; for library, libraryInfo in sorted_libraries_by_insert: read1 =libraryInfo["pair1"] read2 =libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] if orientation=="innie": path,file=os.path.split(read1) file = file.replace("_1.fastq", "_?.fastq") inGroups_file.write("PE{}, lib{}, {}\n".format(group_name, insert, os.path.join(path, file))) group_name += 1 if insert not in librariesForInLibsDict: librariesForInLibsDict[insert] = insert librariesForInLibs.append("lib{}, genome, genome, fragment, 1, {}, {}, , , inward, 0, 0\n".format(insert,insert, std)) elif orientation=="outtie": path,file=os.path.split(read1) file = file.replace("_1.fastq", "_?.fastq") inGroups_file.write("MP{}, lib{}, {}\n".format(group_name, insert, os.path.join(path, file))) group_name += 1 if insert not in librariesForInLibsDict: librariesForInLibsDict[insert] = insert librariesForInLibs.append("lib{}, genome, genome, fragment, 1, , , {}, {}, outward, 0, 0\n".format(insert,insert, std)) else: print "all paths support only innies and outties" inGroups_file.close() for lib in librariesForInLibs: inLibs_file.write(lib) inLibs_file.close() #NOW RUN ALLPATHS FOR REAL program=os.path.join(programBIN, "PrepareAllPathsInputs.pl") os.mkdir("data_dir") data_dir = os.path.join(os.getcwd(), "data_dir") ploidy = "PLOIDY=1" if len(program_options) > 0: if len(program_options) >1: print "Running ALlpaths only one parameter accepted as option here: PLOIDY=2" return sample_config if program_options[0] == "PLOIDY=2": ploidy = "PLOIDY=2" else: print "Running ALlpaths only one parameter accepted as option here: PLOIDY=2" return sample_config command = [program , "DATA_DIR={}".format(data_dir), ploidy, "PICARD_TOOLS_DIR={}".format(global_config["Tools"]["picard"]["bin"]), "FORCE_PHRED=True", "PHRED_64=False"] if common.check_dryrun(sample_config): common.print_command(command) program = os.path.join(programBIN, "RunAllPathsLG") command = [program, "PRE=.", "REFERENCE_NAME=.", "DATA_SUBDIR=data_dir", "RUN=allpaths", "SUBDIR=run"] ommon.print_command(command) return sample_config assembler_stdOut = open("allpaths_PrepareAllPathsInputs.stdOut", "w") assembler_stdErr = open("allpaths_PrepareAllPathsInputs.stdErr", "w") common.print_command(command) returnValue = subprocess.call(command, stdout=assembler_stdOut, stderr=assembler_stdErr) assembler_stdOut.close() assembler_stdErr.close() if returnValue == 0: program = os.path.join(programBIN, "RunAllPathsLG") command = [program, "PRE=.", "REFERENCE_NAME=.", "DATA_SUBDIR=data_dir", "RUN=allpaths", "SUBDIR=run"] common.print_command(command) assembler_stdOut = open("allpaths_RunAllPathsLG.stdOut", "w") assembler_stdErr = open("allpaths_RunAllPathsLG.stdErr", "w") returnValue = subprocess.call(command, stdout=assembler_stdOut, stderr=assembler_stdErr) if returnValue != 0: print "ALLPATHS RunAllPathsLG terminated with an error. Please check running folder for more informations" os.chdir("..") return sample_config else: # save results assembly_dir = os.path.join("data_dir", "allpaths", "ASSEMBLIES", "run") if os.path.exists(os.path.join(assembly_dir, "final.assembly.fasta" )): subprocess.call(["cp", os.path.join(assembly_dir, "final.contigs.fasta" ), "{}.ctg.fasta".format(outputName) ]) subprocess.call(["cp", os.path.join(assembly_dir, "final.assembly.fasta" ), "{}.scf.fasta".format(outputName) ]) else: print "something wrong with Allpaths -> no contig file generated" oc.chdir("..") return sample_config else: print "ALLPATHS PrepareAllPathInputs terminated with an error. Please check running folder for more informations" oc.chdir("..") return sample_config os.chdir("..") return sample_config
def _run_spades(global_config, sample_config, sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "spades" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in cabog case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure(assembler, assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config ########### HERE IT START THE SPECIFIC ASSEMBLER PART command = "" command += "{} ".format(programBIN) for option in program_options: command += "{} ".format(option) #creates the command on-the-fly peLibrary = 1 mpLibrary = 1 for library, libraryInfo in sorted_libraries_by_insert: read1 = libraryInfo["pair1"] read2 = libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] if orientation=="innie" or orientation=="none": if read2 is None: command += "--pe{}-s {} ".format(peLibrary, read1) else: command += "--pe{}-1 {} --pe{}-2 {} ".format(peLibrary, read1, peLibrary, read2) peLibrary += 1 elif orientation=="outtie": command += "--mp{}-1 {} --mp{}-2 {} ".format(mpLibrary, read1, mpLibrary, read2) mpLibrary += 1 else: print "orientation{} not supported.... why the program didnot failed earlier?".format(orientation) command += "-o {} ".format(outputName) common.print_command(command) returnValue = 0 if not common.check_dryrun(sample_config): assembler_stdOut = open("spades.stdOut", "a") assembler_stdErr = open("spades.stdErr", "a") returnValue = subprocess.call(command, stdout=assembler_stdOut, stderr=assembler_stdErr, shell=True) else: return sample_config if returnValue == 0: if os.path.exists(os.path.join(outputName,"contigs.fasta")): subprocess.call(["cp", os.path.join(outputName,"contigs.fasta"), "{}.ctg.fasta".format(outputName)]) subprocess.call(["cp", os.path.join(outputName,"scaffolds.fasta"), "{}.scf.fasta".format(outputName)]) subprocess.call(["rm", "-r", outputName]) else: print "something wrong with SPADES -> no contig file generated" else: print "SPADES terminated with an error. Please check running folder for more informations" os.chdir("..") return sample_config
def _run_soapdenovo(global_config, sample_config, sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "soapdenovo" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in cabog case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure(assembler, assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config ########### HERE IT START THE SPECIFIC ASSEMBLER PART kmer = 54 if "kmer" in sample_config: kmer = sample_config["kmer"] threads = ["-p", "8"] # default for UPPMAX if "threads" in sample_config: threads = ["-p", "{}".format(sample_config["threads"])] soap_config_file = open("configuration.txt", "w") soap_config_file.write("max_rd_len=100\n") #TODO make this a parameter in the options rank = 1 for library, libraryInfo in sorted_libraries_by_insert: soap_config_file.write("[LIB]\n") read1 =libraryInfo["pair1"] read2 =libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] soap_config_file.write("avg_ins={}\n".format(insert)) soap_config_file.write("rank={}\n".format(rank)) rank += 1 soap_config_file.write("map_len=30\n") if orientation=="innie" or orientation=="none": soap_config_file.write("asm_flags=3\n") soap_config_file.write("pair_num_cutoff=3\n") soap_config_file.write("reverse_seq=0\n") if read2 is None: soap_config_file.write("q={}\n".format(read1)) else: soap_config_file.write("q1={}\n".format(read1)) soap_config_file.write("q2={}\n".format(read2)) elif orientation=="outtie": soap_config_file.write("asm_flags=2\n") soap_config_file.write("pair_num_cutoff=5\n") soap_config_file.write("reverse_seq=1\n") soap_config_file.write("q1={}\n".format(read1)) soap_config_file.write("q2={}\n".format(read2)) soap_config_file.close() assembler_stdOut = open("soap.stdOut", "w") assembler_stdErr = open("soap.stdErr", "w") os.makedirs(os.path.join(assemblyDirectory, "runSOAP")) os.chdir("runSOAP") #TODO : lots of missing options command = [programBIN , "all", "-s", "../configuration.txt", "-K", "{}".format(kmer), "-L", "500", "-o", "soapAssembly", threads[0] , threads[1] ] common.print_command(command) returnValue = 0 if not common.check_dryrun(sample_config): subprocess.call(command, stdout=assembler_stdOut, stderr=assembler_stdErr) else: return sample_config os.chdir("..") if returnValue == 0: if(os.path.exists(os.path.join("runSOAP","soapAssembly.scafSeq"))): subprocess.call(["mv", os.path.join("runSOAP","soapAssembly.scafSeq"), "{}.scf.fasta".format(outputName) ]) subprocess.call(["mv", os.path.join("runSOAP","soapAssembly.contig"), "{}.ctg.fasta".format(outputName) ]) subprocess.call(["rm", "-r", "runSOAP"]) else: print "something wrong with SOAPdenovo -> no contig file generated" else: print "SOAPdenovo terminated with an error. Please check running folder for more informations" oc.chdir("..") return sample_config os.chdir("..") return sample_config
def _run_masurca(global_config, sample_config,sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "masurca" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in cabog case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure(assembler, assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config ########### HERE IT START THE SPECIFIC ASSEMBLER PART masurca_config_file = open("configuration.txt", "w") masurca_config_file.write("PATHS\n") masurca_config_file.write("JELLYFISH_PATH=" + os.path.join("{}".format(programBIN), "bin" ) + "\n" ) masurca_config_file.write("SR_PATH=" + os.path.join("{}".format(programBIN), "bin" ) + "\n" ) masurca_config_file.write("CA_PATH=" + os.path.join("{}".format(programBIN), "CA/Linux-amd64/bin") + "\n") masurca_config_file.write("END\n") masurca_config_file.write("\n") masurca_config_file.write("DATA\n") allTheLetters = string.lowercase libraryPE = "p" libraryPEnum = 0 libraryMP = "m" libraryMPnum = 0 #TODO: single ended reads for library, libraryInfo in sorted_libraries_by_insert: read1=libraryInfo["pair1"] read2=libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] if orientation=="innie": if read2 is not None: configurationLine = "PE = {}{} {} {} {} {}".format(libraryPE, allTheLetters[libraryPEnum], insert, std, read1, read2) masurca_config_file.write("{}\n".format(configurationLine)) libraryPEnum+=1 #TODO: check when more than 21 PE libraries ae specified elif orientation=="outtie": configurationLine = "JUMP = {}{} {} {} {} {}".format(libraryMP, allTheLetters[libraryMPnum], insert, std, read1, read2) masurca_config_file.write("{}\n".format(configurationLine)) libraryMPnum += 1 #TODO: check when more than 21 PE libraries ae specified masurca_config_file.write("END\n") masurca_config_file.write("\n") masurca_config_file.write("PARAMETERS\n") #this is k-mer size for deBruijn graph values between 25 and 101 are supported, auto will compute the optimal size based on the read data and GC content masurca_config_file.write("GRAPH_KMER_SIZE=auto\n") #set this to 1 for Illumina-only assemblies and to 0 if you have 2x or more long (Sanger, 454) reads masurca_config_file.write("USE_LINKING_MATES=1\n") #this parameter is useful if you have too many jumping library mates. Typically set it to 60 for bacteria and something large (300) for mammals masurca_config_file.write("LIMIT_JUMP_COVERAGE = 60\n") #these are the additional parameters to Celera Assembler. do not worry about performance, number or processors or batch sizes -- these are computed automatically. for mammals do not set cgwErrorRate above 0.15!!! masurca_config_file.write("CA_PARAMETERS = ovlMerSize=30 cgwErrorRate=0.25 ovlMemory=4GB\n") #minimum count k-mers used in error correction 1 means all k-mers are used. one can increase to 2 if coverage >100 masurca_config_file.write("KMER_COUNT_THRESHOLD = 1\n") #auto-detected number of cpus to use masurca_config_file.write("NUM_THREADS= 8\n") #this is mandatory jellyfish hash size masurca_config_file.write("JF_SIZE=1000000000\n") #this specifies if we do (1) or do not (0) want to trim long runs of homopolymers (e.g. GGGGGGGG) from 3' read ends, use it for high GC genomes masurca_config_file.write("DO_HOMOPOLYMER_TRIM=0\n") masurca_config_file.write("END\n") masurca_config_file.write("\n") masurca_config_file.close() masurca_stdOut = open("masurca.stdOut", "w") masurca_stdErr = open("masurca.stdErr", "w") os.mkdir("runMASURCA") os.chdir("runMASURCA") command = [os.path.join(programBIN,"bin/runSRCA.pl") , "../configuration.txt"] common.print_command(command) if not common.check_dryrun(sample_config): subprocess.call(command, stdout=masurca_stdOut, stderr=masurca_stdErr) else: return sample_config if not os.path.exists("assemble.sh"): print "MaSuRCA: assemble.sh not created. Unknown failure" return sample_config command = ["./assemble.sh"] common.print_command(command) returnValue = subprocess.call(command, stdout=masurca_stdOut, stderr=masurca_stdErr) os.chdir("..") if returnValue == 0: if os.path.exists(os.path.join("runMASURCA","CA/10-gapclose/genome.scf.fasta")): subprocess.call(["cp", os.path.join("runMASURCA","CA/10-gapclose/genome.ctg.fasta"), "{}.ctg.fasta".format(outputName) ]) subprocess.call(["cp", os.path.join("runMASURCA","CA/10-gapclose/genome.scf.fasta"), "{}.scf.fasta".format(outputName) ]) subprocess.call(["rm", "-r", "runMASURCA"]) else: print "something wrong with MaSuRCA -> no contig file generated" else: print "MaSuRCA terminated with an error. Please check running folder for more informations" return sample_config os.chdir("..") return sample_config
def _run_abyss(global_config, sample_config, sorted_libraries_by_insert): ########## ACQUIRE ALL THE INFO AND CREATE THE ASSEMBLY FOLDER assembler = "abyss" outputName = sample_config["output"] currentDirectory = os.getcwd() assemblyDirectory = os.path.join(currentDirectory, assembler) programBIN = global_config["Tools"][assembler]["bin"] # in abyss case there is no exectuable program_options = global_config["Tools"][assembler]["options"] sorted_libraries_by_insert = common._sort_libraries_by_insert(sample_config) if _prepare_folder_structure("abyss", assemblyDirectory) == 0: os.chdir(assemblyDirectory) else: return sample_config ########### HERE IT START THE SPECIFIC ASSEMBLER PART assembler_stdOut = open("abyss.stdOut", "a") assembler_stdErr = open("abyss.stdErr", "a") program=os.path.join(programBIN, "abyss-pe") command = "" command += "{} ".format(program) threads = 8 # default for UPPMAX if "threads" in sample_config : threads = sample_config["threads"] command += "np={} ".format(threads) kmer = 54 if "kmer" in sample_config: kmer = sample_config["kmer"] command += "k={} ".format(kmer) libraries = {} for library, libraryInfo in sorted_libraries_by_insert: read1 = libraryInfo["pair1"] read2 = libraryInfo["pair2"] orientation = libraryInfo["orientation"] insert = libraryInfo["insert"] std = libraryInfo["std"] if orientation=="innie" or orientation=="none": if read2 is None: if "se" not in libraries: # check if this is the first time I insert a se file libraries["se"] = "se=\'" libraries["se"] = libraries["se"] + read1 else: if not "lib" in libraries: libraries["lib"] = {} libName = insert # lib name is the insert size if not libName in libraries["lib"]: libraries["lib"][libName] = "" libraries["lib"][libName] += "{} {} ".format(read1, read2) else: if not "mp" in libraries: libraries["mp"] = {} libName = format(insert) if not libName in libraries["mp"]: libraries["mp"][libName] = "" libraries["mp"][libName] += "{} {} ".format(read1, read2) #now create the command command += "name={} ".format(outputName) librariesSE = "" librariesPE = "" librariesMP = "" if "se" in libraries: libraries["se"] = libraries["se"] + "\'" librariesSE = libraries["se"] if "lib" in libraries: lib="lib=\'" for libPE, libPEreads in sorted(libraries["lib"].iteritems()): lib = lib + "lib{} ".format(libPE) librariesPE += " lib{}=\'{}\' ".format(libPE,libPEreads) lib=lib + "\' " command += "{} ".format(lib) if "mp" in libraries: mp="mp=\'" for libMP, libMPreads in sorted(libraries["mp"].iteritems()): mp = mp + "lib{} ".format(libMP) librariesMP += " lib{}=\'{}\' ".format(libMP,libMPreads) mp=mp + "\' " command += "{} ".format(mp) command += "{} ".format(librariesSE) command += "{} ".format(librariesPE) command += "{} ".format(librariesMP) os.makedirs(os.path.join(assemblyDirectory, "runABySS")) os.chdir("runABySS") returnValue = 0 common.print_command(command) if not common.check_dryrun(sample_config): returnValue = subprocess.call(command, stdout=assembler_stdOut, stderr=assembler_stdErr, shell=True) os.chdir("..") if returnValue == 0 and not common.check_dryrun(sample_config): if os.path.exists(os.path.join("runABySS","{}-contigs.fa".format(outputName))): subprocess.call(["cp", os.path.join("runABySS","{}-contigs.fa".format(outputName)), "{}.ctg.fasta".format(outputName) ]) subprocess.call(["cp", os.path.join("runABySS","{}-scaffolds.fa".format(outputName)), "{}.scf.fasta".format(outputName) ]) subprocess.call(["rm", "-r", "runABySS"]) elif not common.check_dryrun(sample_config): print "something wrong with ABySS -> no contig file generated" return sample_config else: print "ABySS terminated with an error. Please check running folder for more informations" os.chdir("..") return sample_config