# create tmpdir to store fasta files and output files TMPDIR = 'phobius_' + str(uuid.uuid4()) # split fasta lib.splitFASTA(args.input, TMPDIR) # now get list of files in tmpdir proteins = [] for file in os.listdir(TMPDIR): if file.endswith('.fa'): proteins.append(file) # now run the script if lib.which('phobius.pl'): lib.runMultiProgress(runPhobiusLocal, proteins, multiprocessing.cpu_count()) else: lib.runMultiProgress(runPhobiusRemote, proteins, 29) # max is 30 jobs at a time # collect all results phobius = [] for file in os.listdir(TMPDIR): if file.endswith('.phobius'): phobius.append(os.path.join(TMPDIR, file)) # write output TMdomain = 0 SigPep = 0 with open(args.out, 'w') as output: output.write("%s\t%s\t%s\t%s\n" % ('ID', 'TM', 'SP', 'Prediction'))
SeqIO.write(record, output, 'fasta') else: name = str(record.id) scaffolds.append(name) outputfile = os.path.join(tmpdir, name + '.fa') with open(outputfile, 'w') as output: SeqIO.write(record, output, 'fasta') # now loop through each scaffold running augustus if args.cpus > len(scaffolds): num = len(scaffolds) else: num = args.cpus lib.log.debug("Running Augustus on %i chunks, using %i CPUs" % (len(scaffolds), num)) lib.runMultiProgress(runAugustus, scaffolds, num) lib.log.debug("Augustus prediction is finished, now concatenating results") with open(os.path.join(tmpdir, 'augustus_all.gff3'), 'w') as output: for file in scaffolds: file = os.path.join(tmpdir, file + '.augustus.gff3') with open(file) as input: output.write(input.read()) if lib.checkannotations(os.path.join(tmpdir, 'augustus_all.gff3')): lib.log.debug('Augustus finished, now joining results') if lib.which_path('join_aug_pred.pl'): join_script = 'join_aug_pred.pl' else: join_script = os.path.join(AUGUSTUS_BASE, 'scripts', 'join_aug_pred.pl')
'-e', os.path.join(outputDir, os.path.basename(args.transcripts)) ] if args.repeats: cmd += [ '--repeats', os.path.join(outputDir, os.path.basename(args.repeats)) ] cmd += [ os.path.join(outputDir, 'evm.out'), os.path.join(outputDir, 'evm.out.log') ] file_list.append(cmd) # run runMultiProgress lib.runMultiProgress(safe_run, file_list, num_workers, progress=args.progress) # now combine the paritions cmd4 = [ perl, Combine, '--partitions', os.path.basename(partitions), '--output_file_name', 'evm.out' ] lib.runSubprocess(cmd4, tmpdir, lib.log) # now convert to GFF3 cmd5 = [ perl, Convert, '--partitions', os.path.basename(partitions), '--output', 'evm.out', '--genome', os.path.abspath(args.fasta) ] lib.runSubprocess(cmd5, tmpdir, lib.log)
lib.log.info('Found {0:,}'.format(len(Hits)) + ' preliminary alignments --> aligning with exonerate') # index the genome and proteins # do index here in case memory problems? protein_dict = SeqIO.index(os.path.abspath(args.proteins), 'fasta') # split genome fasta into individual scaffolds with open(os.path.abspath(args.genome), 'rU') as input: for record in SeqIO.parse(input, "fasta"): SeqIO.write(record, os.path.join(tmpdir, 'scaffolds', record.id + ".fa"), "fasta") # run multiprocessing exonerate lib.runMultiProgress(runExonerate, Hits, args.cpus) # now need to loop through and offset exonerate predictions back to whole scaffolds exonerate_raw = os.path.join(tmpdir, 'exonerate.out.combined') with open(exonerate_raw, 'w') as output: for file in os.listdir(tmpdir): if file.endswith('.out'): with open(os.path.join(tmpdir, file), 'rU') as exoresult: offset = int(file.split('__')[1]) for line in itertools.islice(exoresult, 3, None): if line.startswith('#') or line.startswith( 'Average') or line.startswith('-- completed'): output.write(line) else: cols = line.split('\t') cols[3] = str(int(cols[3]) + offset)
def runTrinityGG(genome, readTuple, longReads, shortBAM, output, args=False): ''' function will run genome guided Trinity. First step will be to run hisat2 to align reads to the genome, then pass that BAM file to Trinity to generate assemblies ''' if not lib.checkannotations(shortBAM): # build hisat2 index, using exons and splice sites lib.log.info("Building Hisat2 genome index") cmd = ['hisat2-build', '-p', str(args.cpus), genome, os.path.join(tmpdir, 'hisat2.genome')] lib.runSubprocess4(cmd, '.', lib.log) # align reads using hisat2 lib.log.info("Aligning reads to genome using Hisat2") # use bash wrapper for samtools piping for SAM -> BAM -> sortedBAM # use half number of threads for bam compression threads bamthreads = (args.cpus + 2 // 2) // 2 if args.stranded != 'no' and not readTuple[2]: hisat2cmd = ['hisat2', '-p', str(args.cpus), '--max-intronlen', str(args.max_intronlen), '--dta', '-x', os.path.join(tmpdir, 'hisat2.genome'), '--rna-strandness', args.stranded] else: hisat2cmd = ['hisat2', '-p', str(args.cpus), '--max-intronlen', str(args.max_intronlen), '--dta', '-x', os.path.join(tmpdir, 'hisat2.genome')] if readTuple[0] and readTuple[1]: hisat2cmd = hisat2cmd + ['-1', readTuple[0], '-2', readTuple[1]] if readTuple[2]: hisat2cmd = hisat2cmd + ['-U', readTuple[2]] cmd = [os.path.join(parentdir, 'sam2bam.sh'), " ".join( hisat2cmd), str(bamthreads), shortBAM] lib.runSubprocess(cmd, '.', lib.log) else: lib.log.info('Existig Hisat2 alignments found: {:}'.format(shortBAM)) # now launch Trinity genome guided TrinityLog = os.path.join(tmpdir, 'Trinity-gg.log') lib.log.info("Running genome-guided Trinity, logfile: %s" % TrinityLog) lib.log.info( "Clustering of reads from BAM and preparing assembly commands") jaccard_clip = [] if args.jaccard_clip: jaccard_clip = ['--jaccard_clip'] if args.stranded != 'no': cmd = ['Trinity', '--SS_lib_type', args.stranded, '--no_distributed_trinity_exec', '--genome_guided_bam', shortBAM, '--genome_guided_max_intron', str( args.max_intronlen), '--CPU', str(args.cpus), '--max_memory', args.memory, '--output', os.path.join(tmpdir, 'trinity_gg')] else: cmd = ['Trinity', '--no_distributed_trinity_exec', '--genome_guided_bam', shortBAM, '--genome_guided_max_intron', str( args.max_intronlen), '--CPU', str(args.cpus), '--max_memory', args.memory, '--output', os.path.join(tmpdir, 'trinity_gg')] cmd = cmd + jaccard_clip if longReads and lib.checkannotations(longReads): cmd = cmd + ['--long_reads', os.path.realpath(longReads)] lib.runSubprocess2(cmd, '.', lib.log, TrinityLog) commands = os.path.join(tmpdir, 'trinity_gg', 'trinity_GG.cmds') # this will create all the Trinity commands, will now run these in parallel using multiprocessing # in Python (seems to be much faster than Parafly on my system) file_list = [] with open(commands, 'r') as cmdFile: for line in cmdFile: line = line.replace('\n', '') # don't think this should be appended to every command.... line = line.replace('--no_distributed_trinity_exec', '') line = line.replace('"', '') # don't need these double quotes file_list.append(line) lib.log.info("Assembling "+"{0:,}".format(len(file_list)) + " Trinity clusters using %i CPUs" % (args.cpus-1)) lib.runMultiProgress(safe_run, file_list, args.cpus-1) # collected output files and clean outputfiles = os.path.join( tmpdir, 'trinity_gg', 'trinity_output_files.txt') with open(outputfiles, 'w') as fileout: for filename in find_files(os.path.join(tmpdir, 'trinity_gg'), '*inity.fasta'): fileout.write('%s\n' % filename) # now grab them all using Trinity script cmd = ['perl', os.path.abspath(os.path.join( TRINITY, 'util', 'support_scripts', 'GG_partitioned_trinity_aggregator.pl')), 'Trinity_GG'] lib.runSubprocess5(cmd, '.', lib.log, outputfiles, output) lib.log.info('{:,} transcripts derived from Trinity'.format( lib.countfasta(output)))
SeqIO.write(record, output, 'fasta') else: name = str(record.id) scaffolds.append(name) outputfile = os.path.join(tmpdir, name + '.fa') with open(outputfile, 'w') as output: SeqIO.write(record, output, 'fasta') # now loop through each scaffold running augustus if args.cpus > len(scaffolds): num = len(scaffolds) else: num = args.cpus lib.log.debug("Running Augustus on %i chunks, using %i CPUs" % (len(scaffolds), num)) lib.runMultiProgress(runAugustus, scaffolds, num, progress=args.progress) lib.log.debug("Augustus prediction is finished, now concatenating results") with open(os.path.join(tmpdir, 'augustus_all.gff3'), 'w') as output: for file in scaffolds: file = os.path.join(tmpdir, file + '.augustus.gff3') with open(file) as input: output.write(input.read()) if lib.checkannotations(os.path.join(tmpdir, 'augustus_all.gff3')): lib.log.debug('Augustus finished, now joining results') if lib.which_path('join_aug_pred.pl'): join_script = 'join_aug_pred.pl' else: join_script = os.path.join(AUGUSTUS_BASE, 'scripts', 'join_aug_pred.pl')
'-e', os.path.join(outputDir, os.path.basename(args.transcripts)) ] if args.repeats: cmd += [ '--repeats', os.path.join(outputDir, os.path.basename(args.repeats)) ] cmd += [ os.path.join(outputDir, 'evm.out'), os.path.join(outputDir, 'evm.out.log') ] file_list.append(cmd) # run runMultiProgress lib.runMultiProgress(safe_run, file_list, num_workers) # now combine the paritions cmd4 = [ perl, Combine, '--partitions', os.path.basename(partitions), '--output_file_name', 'evm.out' ] lib.runSubprocess(cmd4, tmpdir, lib.log) # now convert to GFF3 cmd5 = [ perl, Convert, '--partitions', os.path.basename(partitions), '--output', 'evm.out', '--genome', os.path.abspath(args.fasta) ] lib.runSubprocess(cmd5, tmpdir, lib.log)