def epcr(self): while True: # I think this should work for getting output from processes and writing to a logfile - but it's long # and terrible - maybe possible to get it set up in accessoryFunctions so it's a bit less of a pain? # Setup a threadlock for later so multiple processes don't all try to write their output at once. threadlock = threading.Lock() # Get our stdout and stderr strings set up. outstr = '' errstr = '' sample, linkfile = self.epcrqueue.get() if not os.path.isfile('{}.famap'.format(linkfile)): # Run the subprocess, then get the stdout in outstr and stderr in errstr out, err = run_subprocess(sample.commands.famap) outstr += out errstr += err if not os.path.isfile('{}.hash'.format(linkfile)): out, err = run_subprocess(sample.commands.fahash) outstr += out errstr += err if not os.path.isfile('{}.txt'.format(linkfile)): out, err = run_subprocess(sample.commands.epcr) outstr += out errstr += err # Once processes are finished running, get the threadlock, because now it's output writing time. threadlock.acquire() # Write stuff to the logfile. write_to_logfile(sample.commands.famap, sample.commands.famap, self.logfile) write_to_logfile(sample.commands.fahash, sample.commands.fahash, self.logfile) write_to_logfile(sample.commands.epcr, sample.commands.epcr, self.logfile) write_to_logfile(outstr, errstr, self.logfile) threadlock.release() # Release the threadlock so that other processes can get on with it. self.epcrqueue.task_done()
def run_jellyfish(self): """ Runs jellyfish to split subsample reads into kmers. Runs kmers through a bloom filter to get rid of singletons that are likely just sequencing errors. Should be run after subsampling reads. """ for sample in self.metadata: # Set the name of the jellyfish count file sample[self.analysistype].jellyfish_file = os.path.join( sample[self.analysistype].outputdir, sample.name + '_jellyfish') # Set the system call sample[self.analysistype].jellyfishcountcmd \ = 'jellyfish count -m 31 -s 100M --bf-size 100M -C -F 2 {} -o {} -t {}'\ .format(sample[self.analysistype].subsampledreads, sample[self.analysistype].jellyfish_file, str(self.threads)) # Run the call, and write any errors to the logfile command = sample[self.analysistype].jellyfishcountcmd if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr)
def bbduker(self): """Run bbduk system calls""" while True: # while daemon # Unpack the variables from the queue (sample, systemcall, reversename) = self.trimqueue.get() # Check to see if the forward file already exists if systemcall: threadlock = threading.Lock() if not os.path.isfile(reversename) and not os.path.isfile( '{}.bz2'.format(reversename)): # Run the call out, err = run_subprocess(systemcall) threadlock.acquire() write_to_logfile(systemcall, systemcall, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None) threadlock.release() # Define the output directory outputdir = sample.general.outputdirectory # Add the trimmed fastq files to a list trimmedfastqfiles = sorted( glob(os.path.join(outputdir, '*trimmed.fastq.gz'))) # Populate the metadata if the files exist sample.general.trimmedfastqfiles = trimmedfastqfiles if trimmedfastqfiles else 'NA' # Signal to trimqueue that job is done self.trimqueue.task_done()
def predict(self): while True: sample = self.predictqueue.get() # Populate attributes sample.prodigal.reportdir = os.path.join( sample.general.outputdirectory, 'prodigal') sample.prodigal.results_file = os.path.join( sample.prodigal.reportdir, '{}_prodigalresults.sco'.format(sample.name)) sample.prodigal.results = sample.prodigal.results_file sample.commands.prodigal = 'prodigal -i {in1} -o {out1} -f sco -d {genes}'\ .format(in1=sample.general.bestassemblyfile, out1=sample.prodigal.results_file, genes=os.path.join(sample.prodigal.reportdir, '{}_genes.fa'.format(sample.name))) # Create the folder to store the reports make_path(sample.prodigal.reportdir) # Determine if the report already exists, and that it is not empty size = 0 if os.path.isfile(sample.prodigal.results_file): size = os.stat(sample.prodigal.results_file).st_size if not os.path.isfile(sample.prodigal.results_file) or size == 0: # Run the command out, err = run_subprocess(sample.commands.prodigal) threadlock.acquire() write_to_logfile(sample.commands.prodigal, sample.commands.prodigal, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None) threadlock.release() self.predictqueue.task_done()
def assemble(self): """Run the assembly command in a multi-threaded fashion""" threadlock = threading.Lock() while True: (sample, command) = self.assemblequeue.get() if command and not os.path.isfile( os.path.join(sample.general.spadesoutput, 'contigs.fasta')): # execute(command) out, err = run_subprocess(command) threadlock.acquire() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None) threadlock.release() # call(command, shell=True, stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb')) dotter() # Signal to the queue that the job is done self.assemblequeue.task_done()
def subsample_reads(self): """ Subsampling of reads to 20X coverage of rMLST genes (roughly). To be called after rMLST extraction and read trimming, in that order. """ for sample in self.metadata: # Create the name of the subsampled read file sample[self.analysistype].subsampledreads = os.path.join( sample[self.analysistype].outputdir, '{}_targetMatches_subsampled.fastq'.format(self.analysistype)) # Set the reformat.sh command - as this command will be run multiple times, overwrite previous iterations # each time. Use samplebasestarget to provide an approximation of the number of bases to include in the # subsampled reads e.g. for rMLST: 700000 (approx. 35000 bp total length of genes x 20X coverage) sample[self.analysistype].subsamplecmd = 'reformat.sh in={} out={} overwrite samplebasestarget={}' \ .format(sample[self.analysistype].baitedfastq, sample[self.analysistype].subsampledreads, self.samplebasestarget) # Run the call, and write any errors to the logfile command = sample[self.analysistype].subsamplecmd if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr)
def run_bbmap(self): """ Runs bbmap on kmer fasta file, against kmer fasta file to generate a samfile which can then be parsed to find low frequency kmers that have one mismatch to high frequency kmers, indicating that they're from contaminating alleles. """ for sample in self.metadata: # Create the name for the output bam file sample[self.analysistype].bamfile = sample[ self.analysistype].mer_fasta.replace('.fasta', '.bam') # Set the bbmap call - use the overwrite option to overwrite previous files that were created on previous # iterations, ambig=all to use all highest scoring mappings, nodisk to build index in memory, and only write # ouput to disk, local to allow soft-clipping sample[self.analysistype].bbmapcmd = \ 'bbmap.sh ref={} in={} outm={} overwrite ambig=all nodisk local threads={}'\ .format(sample[self.analysistype].solid_mers, sample[self.analysistype].solid_mers, sample[self.analysistype].bamfile, str(self.threads)) # Run the call, and write any errors to the logfile command = sample[self.analysistype].bbmapcmd if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr)
def makeblastdb(self): """ Makes blast database files from targets as necessary """ # Iterate through the samples to set the bait file. for sample in self.runmetadata.samples: if sample.general.bestassemblyfile != 'NA': # Remove the file extension db = os.path.splitext(sample[self.analysistype].baitfile)[0] # Add '.nhr' for searching below nhr = '{}.nhr'.format(db) # Check for already existing database files if not os.path.isfile(str(nhr)): # Create the databases command = 'makeblastdb -in {} -parse_seqids -max_file_sz 2GB -dbtype nucl -out {}'\ .format(sample[self.analysistype].baitfile, db) out, err = run_subprocess(command) write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr)
def fastqc(self): """Run fastqc system calls""" while True: # while daemon threadlock = threading.Lock() # Unpack the variables from the queue (sample, systemcall, outputdir, fastqcreads) = self.qcqueue.get() # Check to see if the output HTML file already exists try: _ = glob(os.path.join(outputdir, '*.html'))[0] except IndexError: # Make the output directory make_path(outputdir) # Run the system calls outstr = str() errstr = str() out, err = run_subprocess(systemcall) outstr += out errstr += err out, err = run_subprocess(fastqcreads) outstr += out errstr += err # Acquire thread lock, and write the logs to file threadlock.acquire() write_to_logfile(systemcall, systemcall, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(fastqcreads, fastqcreads, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(outstr, errstr, self.logfile, sample.general.logout, sample.general.logerr, None, None) threadlock.release() # Rename the outputs try: shutil.move( os.path.join(outputdir, 'stdin_fastqc.html'), os.path.join(outputdir, '{}_fastqc.html'.format(sample.name))) shutil.move( os.path.join(outputdir, 'stdin_fastqc.zip'), os.path.join(outputdir, '{}_fastqc.zip'.format(sample.name))) except IOError: pass # Signal to qcqueue that job is done self.qcqueue.task_done()
def write_mer_file(self): """ Writes the mer file created by jellyfish to a fasta format to be used by other things downstream. Only writes kmers that have been seen at least twice to attempt to get rid of sequencing erros. """ for sample in self.metadata: # Set the name of the kmer file dumped from jellyfish sample[self.analysistype].mer_fasta = sample[ self.analysistype].jellyfish_file + '.fasta' sample[self.analysistype].solid_mers = sample[ self.analysistype].jellyfish_file + '_solid.fasta' # Set the system call sample[self.analysistype].jellyfishdumpcmd =\ 'jellyfish dump {} > {}'\ .format(sample[self.analysistype].jellyfish_file, sample[self.analysistype].mer_fasta) # Run the system call command = sample[self.analysistype].jellyfishdumpcmd if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) # Read in the dumped file to a list with open(sample[self.analysistype].mer_fasta, 'r') as mers: fastas = mers.readlines() # Initialise variables for use in parsing outputs num_mers = 0 sequences = list() # Iterate through the list of the fasta outputs. Output is a multifasta e.g.: # >8 # GCCTGGAAAACTGGCCACCGGCAAGCATCGC # where the header, >8, indicates that the sequence is present 8 times in the sample for i in range(len(fastas)): # Find the headers if '>' in fastas[i]: # If the number of times the sequence is present in the sample is greater than one, increment # the total number of kmers observed if int(fastas[i].replace('>', '')) > 1: num_mers += 1 # Append a string of the header plus the total number of mers, and the sequence information # to the list of sequences e.g. ['>8_1\nGCCTGGAAAACTGGCCACCGGCAAGCATCGC\n'] sequences.append(fastas[i].rstrip() + '_' + str(num_mers) + '\n' + fastas[i + 1]) # Write out our solid kmers to file to be used later. with open(sample[self.analysistype].solid_mers, 'w') as solidmers: solidmers.write(''.join(sequences)) # Update the number of unique kmers if num_mers > sample[self.analysistype].unique_kmers: sample[self.analysistype].unique_kmers = num_mers
def epcr(self): while True: sample, linkfile = self.epcrqueue.get() # Set the names of the output files sample[self.analysistype].famap = '{}.famap'.format(linkfile) sample[self.analysistype].hash = '{}.hash'.format(linkfile) sample[self.analysistype].output = '{}.txt'.format(linkfile) # Initialise a list to store the results sample[self.analysistype].epcrresults = list() # If the files created by the results do not exist, run the necessary system calls threadlock = threading.Lock() # Get our stdout and stderr strings set up. outstr = '' errstr = '' sample, linkfile = self.epcrqueue.get() if not os.path.isfile(sample[self.analysistype].famap): # Run the subprocess, then get the stdout in outstr and stderr in errstr out, err = run_subprocess(sample.commands.famap) outstr += out errstr += err if not os.path.isfile(sample[self.analysistype].famap): out, err = run_subprocess(sample.commands.fahash) outstr += out errstr += err if not os.path.isfile(sample[self.analysistype].output): out, err = run_subprocess(sample.commands.epcr) outstr += out errstr += err # Once processes are finished running, get the threadlock, because now it's output writing time. threadlock.acquire() # Write stuff to the logfile. write_to_logfile(sample.commands.famap, sample.commands.famap, self.logfile) write_to_logfile(sample.commands.fahash, sample.commands.fahash, self.logfile) write_to_logfile(sample.commands.epcr, sample.commands.epcr, self.logfile) write_to_logfile(outstr, errstr, self.logfile) threadlock.release() # Read the results into a list with open(sample[self.analysistype].output, 'r') as results: for line in results: sample[self.analysistype].epcrresults.append(line.strip()) self.epcrqueue.task_done()
def extract_rmlst_reads(self): """ rMLST read extraction. Should be the first thing called after parsing the fastq directory. """ for sample in self.metadata: # Create the object to store the variables setattr(sample, self.analysistype, GenObject()) # Initialise variables sample[self.analysistype].snv_count = list() # Initialise a starting value for the number of unique kmers found in each sample sample[self.analysistype].unique_kmers = -1 # Set and create the output directory try: sample[self.analysistype].outputdir = os.path.join( sample.run.outputdirectory, self.analysistype) except KeyError: sample[self.analysistype].outputdir = os.path.join( sample.general.outputdirectory, self.analysistype) make_path(sample[self.analysistype].outputdir) sample[self.analysistype].logout = os.path.join( sample[self.analysistype].outputdir, 'logout.txt') sample[self.analysistype].logerr = os.path.join( sample[self.analysistype].outputdir, 'logerr.txt') sample[self.analysistype].baitedfastq = os.path.join( sample[self.analysistype].outputdir, '{}_targetMatches.fastq.gz'.format(self.analysistype)) # Create the command to run the baiting - paired inputs and a single, zipped output sample[self.analysistype].bbdukcmd = 'bbduk.sh ref={} in1={} in2={} threads={} outm={}'\ .format(self.database, sample.general.trimmedcorrectedfastqfiles[0], sample.general.trimmedcorrectedfastqfiles[1], str(self.threads), sample[self.analysistype].baitedfastq) # Sometimes bbduk hangs forever, so that needs to be handled. Give it a very generous timeout. try: # Run the call, and write any errors to the logfile command = sample[self.analysistype].bbdukcmd if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) except TimeoutExpired: print('ERROR: Could not extract rMLST reads from sample {}'. format(sample.name))
def mash(self): while True: sample = self.mashqueue.get() if not os.path.isfile(sample[self.analysistype].mashresults): threadlock = threading.Lock() out, err = run_subprocess(sample.commands.mash) threadlock.acquire() write_to_logfile(sample.commands.mash, sample.commands.mash, self.logfile) write_to_logfile(out, err, self.logfile) threadlock.release() # call(sample.commands.mash, shell=True, stdout=self.fnull, stderr=self.fnull) self.mashqueue.task_done()
def sistr(self): """Perform sistr analyses on Salmonella""" printtime('Performing sistr analyses', self.start) for sample in self.metadata: # Create the analysis-type specific attribute setattr(sample, self.analysistype, GenObject()) if sample.general.bestassemblyfile != 'NA': try: # Only process strains that have been determined to be Salmonella if sample.general.referencegenus == 'Salmonella': # Set and create the path of the directory to store the strain-specific reports sample[self.analysistype].reportdir = os.path.join( sample.general.outputdirectory, self.analysistype) # Name of the .json output file sample[self.analysistype].jsonoutput = os.path.join( sample[self.analysistype].reportdir, '{}.json'.format(sample.name)) # Set the sistr system call sample.commands.sistr = \ 'sistr -f json -o {} -t {} -T {} {}'\ .format(sample[self.analysistype].jsonoutput, self.cpus, os.path.join(sample[self.analysistype].reportdir, 'tmp'), sample.general.bestassemblyfile) # sample[self.analysistype].logout = os.path.join( sample[self.analysistype].reportdir, 'logout') sample[self.analysistype].logerr = os.path.join( sample[self.analysistype].reportdir, 'logerr') # Only run the analyses if the output json file does not exist if not os.path.isfile( sample[self.analysistype].jsonoutput): out, err = run_subprocess(sample.commands.sistr) write_to_logfile(sample.commands.sistr, sample.commands.sistr, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) self.queue.task_done() except (ValueError, KeyError): pass self.queue.join() self.report()
def assemble(self): while True: sample = self.assemblequeue.get() if not os.path.isfile(sample.general.assemblyfile): # Run the assembly out, err = run_subprocess(sample.commands.assemble) self.threadlock.acquire() write_to_logfile(sample.commands.assemble, sample.commands.assemble, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None) self.threadlock.release() self.assemblequeue.task_done()
def makeblastdb(self, fastapath): """ Makes blast database files from targets as necessary """ # remove the path and the file extension for easier future globbing db = fastapath.split('.')[0] nhr = '{}.nhr'.format(db) # add nhr for searching if not os.path.isfile(str(nhr)): # if check for already existing dbs # Create the databases threadlock = threading.Lock() command = 'makeblastdb -in {} -parse_seqids -max_file_sz 2GB -dbtype nucl -out {}'.format(fastapath, db) out, err = run_subprocess(command) threadlock.acquire() write_to_logfile(out, err, self.logfile) threadlock.release() dotter()
def run_qaml(self): """ Create and run the GenomeQAML system call """ printtime('Running GenomeQAML quality assessment', self.start) qaml_call = 'classify.py -t {tf} -r {rf}'\ .format(tf=self.qaml_path, rf=self.qaml_report) make_path(self.reportpath) # Only attempt to assess assemblies if the report doesn't already exist if not os.path.isfile(self.qaml_report): # Run the system calls out, err = run_subprocess(qaml_call) # Acquire thread lock, and write the logs to file self.threadlock.acquire() write_to_logfile(qaml_call, qaml_call, self.logfile) write_to_logfile(out, err, self.logfile) self.threadlock.release()
def fastathreads(self): while True: sample = self.fastaqueue.get() # Check to see if the FASTA file already exists if not os.path.isfile(sample[self.analysistype].fasta): # Run the system call , stdout=self.devnull, stderr=self.devnull out, err = run_subprocess(sample[self.analysistype].fastxcall) write_to_logfile(sample[self.analysistype].fastxcall, sample[self.analysistype].fastxcall, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) self.fastaqueue.task_done()
def subsamplethreads(self): while True: sample = self.samplequeue.get() # Check to see if the subsampled FASTQ file has already been created if not os.path.isfile(sample[self.analysistype].subsampledfastq): # Run the system call # call(sample[self.analysistype].seqtkcall, shell=True, stdout=self.devnull, stderr=self.devnull) out, err = run_subprocess(sample[self.analysistype].seqtkcall) write_to_logfile(sample[self.analysistype].seqtkcall, sample[self.analysistype].seqtkcall, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, sample[self.analysistype].logout, sample[self.analysistype].logerr) self.samplequeue.task_done()
def make_db(self): """ Makes the blast database if it isn't present. Doesn't do anything if we already have database files. """ db_files = ['.nhr', '.nin', '.nsq'] db_present = True for db_file in db_files: if not os.path.isfile(self.database + db_file): db_present = False if not db_present: printtime('Making database!', self.start) command = 'makeblastdb -dbtype nucl -in ' + self.database if self.analyse: out, err = run_subprocess(command) else: out = str() err = str() write_to_logfile(command, command, self.logfile, None, None, None, None) write_to_logfile(out, err, self.logfile, None, None, None, None)
def database_download(self, targetcall, databasepath, complete=True): """ Checks to see if the database has already been downloaded. If not, downloads the database, and writes stdout and stderr to the logfile :param targetcall: system call to download, and possibly set-up the database :param databasepath: absolute path of the database :param complete: boolean variable to determine whether the complete file should be created """ # Create a file to store the logs; it will be used to determine if the database was downloaded and set-up completefile = os.path.join(databasepath, 'complete') # Run the system call if the database is not already downloaded if not os.path.isfile(completefile): out, err = run_subprocess(targetcall) print(out, err) # Write the out and err streams to the master files write_to_logfile(out, err, self.logfile, None, None, None, None) if complete: # Create the database completeness assessment file and populate it with the out and err streams with open(completefile, 'w') as complete: complete.write(out) complete.write(err)
def runquast(self): while True: sample, quastoutputdirectory = self.quastqueue.get() make_path(quastoutputdirectory) threadlock = threading.Lock() # fnull = open(os.devnull, 'wb') # Don't re-perform the analysis if the report file exists if not os.path.isfile( '{}/report.tsv'.format(quastoutputdirectory)): out, err = run_subprocess(sample.commands.quast) # call(sample.commands.quast, shell=True, stdout=fnull, stderr=fnull) threadlock.acquire() write_to_logfile(sample.commands.quast, sample.commands.quast, self.logfile, sample.general.logout, sample.general.logerr, None, None) write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None) threadlock.release() # Following the analysis, parse the report (if it exists) into the metadata object if os.path.isfile('{}/report.tsv'.format(quastoutputdirectory)): self.metaparse(sample, quastoutputdirectory) self.quastqueue.task_done()
def createfastq(self): """Uses bcl2fastq to create .fastq files from a MiSeqRun""" # Initialise samplecount samplecount = 0 # If the fastq destination folder is not provided, make the default value of :path/:miseqfoldername self.fastqdestination = self.fastqdestination if self.fastqdestination else self.path + self.miseqfoldername # Make the path make_path(self.fastqdestination) # Initialise variables for storing index information index = '' indexlength = int() # bcl2fastq requires an older version of the sample sheet, this recreates the required version # Create the new sample sheet with open('{}/SampleSheet_modified.csv'.format(self.fastqdestination), "w") as modifiedsamplesheet: # Write the required headings to the file modifiedsamplesheet.write( "FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,Operator,SampleProject\n" ) for strain in self.samples: # Create a combined index of index1-index2 try: strain.run.modifiedindex = '{}-{}'.format( strain.run.index, strain.run.index2) indexlength = 16 index = 'I8,I8' except KeyError: strain.run.modifiedindex = strain.run.index indexlength = 6 index = 'I6' # The list of items to print to each line of the modified sample sheet printlist = [ self.flowcell, '1', strain.name, str(strain.run.SampleNumber), strain.run.modifiedindex, strain.run.Description, 'N', 'NA', strain.run.InvestigatorName, self.projectname ] modifiedsamplesheet.write('{}\n'.format(",".join(printlist))) samplecount += 1 # Set :forward/reverse length to :header.forward/reverse length if the argument is not provided, or it's 'full', # otherwise use the supplied argument self.forwardlength = self.metadata.header.forwardlength if self.forwardlength.lower()\ == 'full' else self.forwardlength # Set :reverselength to :header.reverselength self.reverselength = self.metadata.header.reverselength if self.reverselength.lower() \ == 'full' else self.reverselength # As the number of cycles required is the number of forward reads + the index(8) + the second index(8) # Also set the basemask variable as required if self.reverselength != '0': self.readsneeded = int(self.forwardlength) + int( self.reverselength) + indexlength basemask = "Y{}n*,{},Y{}n*".format(self.forwardlength, index, self.reverselength) nohup = "nohup make -j 16 > nohup.out" else: # + 1 self.readsneeded = int(self.forwardlength) + indexlength basemask = "Y{}n*,{},n*".format(self.forwardlength, index) nohup = "nohup make -j 16 r1 > nohup.out" # Handle plurality appropriately samples = 'samples' if samplecount > 1 else 'sample' number = 'are' if samplecount > 1 else 'is' printtime( 'There {} {} {} in this run. ' 'Running fastq creating module with the following parameters:\n' 'MiSeqPath: {},\n' 'MiSeqFolder: {},\n' 'Fastq destination: {},\n' 'SampleSheet: {}'.format( number, samplecount, samples, self.miseqpath, self.miseqfolder, self.fastqdestination, '{}/SampleSheet_modified.csv'.format(self.fastqdestination)), self.start) # Count the number of completed cycles in the run of interest cycles = glob('{}Data/Intensities/BaseCalls/L001/C*'.format( self.miseqfolder)) while len(cycles) < self.readsneeded: printtime( 'Currently at {} cycles. Waiting until the MiSeq reaches cycle {}' .format(len(cycles), self.readsneeded), self.start) sleep(1800) cycles = glob('{}Data/Intensities/BaseCalls/L001/C*'.format( self.miseqfolder)) # configureBClToFastq requires :self.miseqfolder//Data/Intensities/BaseCalls/config.xml in order to work # When you download runs from BaseSpace, this file is not provided. There is an empty config.xml file that # can be populated with run-specific values and moved to the appropriate folder if not os.path.isfile('{}Data/Intensities/BaseCalls/config.xml'.format( self.miseqfolder)): self.configfilepopulator() # Define the bcl2fastq system call bclcall = "configureBclToFastq.pl --input-dir {}Data/Intensities/BaseCalls " \ "--output-dir {} --force --sample-sheet {}/SampleSheet_modified.csv " \ "--mismatches 1 --no-eamss --fastq-cluster-count 0 --compression none --use-bases-mask {}"\ .format(self.miseqfolder, self.fastqdestination, self.fastqdestination, basemask) # Define the nohup system call nohupcall = "cd {} && {}".format(self.fastqdestination, nohup) # fnull = open(os.devnull, 'wb') if not os.path.isdir("{}/Project_{}".format(self.fastqdestination, self.projectname)): # Call configureBclToFastq.pl printtime('Running bcl2fastq', self.start) # Run the commands threadlock = threading.Lock() outstr = '' outerr = '' out, err = run_subprocess(bclcall) outstr += out outerr += out out, err = run_subprocess(nohupcall) outstr += out outerr += out # call(bclcall, shell=True, stdout=fnull, stderr=fnull) # call(nohupcall, shell=True, stdout=fnull, stderr=fnull) threadlock.acquire() write_to_logfile(bclcall, bclcall, self.logfile) write_to_logfile(nohupcall, nohupcall, self.logfile) write_to_logfile(outstr, outerr, self.logfile) threadlock.release() # Populate the metadata for sample in self.metadata.samples: sample.commands = GenObject() sample.commands.nohup = nohupcall sample.commands.bcl = bclcall sample.run.forwardlength = self.forwardlength sample.run.reverselength = self.reverselength # Copy the fastq files to a central folder so they can be processed self.fastqmover()