def GenerateAAFile(intermediate_file,unique_aa_file,header_row,experiment_names):
#we use bash for calling efficient functions (sort and awk) for processing of a TAB delimited file
#assume the last column of the tab file corresponds to COUNTS for a SPECIFIC EPXERIMENT NUMBER (it has to be a number corersponding to an experiment number)
#these experiment numbers should correlate to indexes in experiment_names variable
total_count_column = len(header_row) #the column for total counts will always be the last column in the header_row variable.
#THEN we have to add counts for each experiment
for e in experiment_names:
header_row.append(e+' Counts')
num_exp_counts = len(experiment_names)
hfile = intermediate_file+'.header.txt'
parent = useful.get_parent_dir(intermediate_file) # os.path.dirname(os.path.dirname(os.path.abspath(intermediate_file)))# '/'.join(intermediate_file.split('/')[:-1])+'/'
#write a header row
with open(hfile,'w') as w:
w.write('\t'.join(header_row)+'\n')
#the bash script process_aa_file will sort sequences in file then count their occurrences and collapse
this_script_folder = useful.get_parent_dir(__file__)
#the bash script,process_unique_ab_aa_file.bash, MUST be in the same folder as this script (immunogrep_descriptive_statistics)
bash_script_path = os.path.join(this_script_folder,'process_unique_ab_aa_file.bash')
bash_command = '''bash '{5}' '{0}' '{1}' '{2}' {3} {4}'''.format(intermediate_file,unique_aa_file,hfile,str(num_exp_counts),str(total_count_column),bash_script_path)
#run the bash
#subprocess.call(bash_command,shell=True)
subprocess.call(bash_command,shell=True)
def run_flash(r1file, r2file, working_directory, outfile='', parameters={}, suffix=''):
r1_path = useful.get_parent_dir(r1file) # '/'.join(r1file.split('/')[:-1])
r2_path = useful.get_parent_dir(r2file) # '/'.join(r2file.split('/')[:-1])
if not parameters:
print "PARAMETERS NOT PASSED INTO FLASH PROGRAM. USING DEFAULT IGSEQ PARAMETERS: R = 300, F = 400"
parameters = {'r': 300, 'f': 400}
if r1file.endswith('.gz'):
print "Unzipping R1 File.."
r1file = useful.gunzip_python(r1file)
if r2file.endswith('.gz'):
print "Unzipping R2 File.."
r2file = useful.gunzip_python(r2file)
working_directory = os.path.abspath(working_directory)
if r1_path != working_directory:
os.rename(r1file, os.path.join(working_directory, os.path.basename(r1file)))
if r2_path != working_directory:
os.rename(r2file, os.path.join(working_directory, os.path.basename(r2file)))
if outfile == '':
outfile = os.path.basename(r1file).split('.')
for p, subs in enumerate(outfile):
if '_R1' in subs:
r_pos = subs.index("_R1")
outfile[p] = subs[:r_pos]
break
elif '_R2' in subs:
r_pos = subs.index("_R2")
outfile[p] = subs[:r_pos]
break
outfile = '.'.join(outfile)
else:
outfile = os.path.basename(outfile)
outfile = outfile.replace('.fastq', '').replace('.fasta', '')
outfile += '.flashed' + suffix
if os.path.isfile(os.path.join(working_directory, outfile)): # in resulting_files:
print('WARNING: FILE {0} ALREADY PRESENT IN FOLDER. FILE WILL BE OVERWRITTEN'.format(working_directory + '/' + outfile))
r1file = os.path.join(working_directory, os.path.basename(r1file)) # working_directory+'/'+os.path.basename(r1file)
r2file = os.path.join(working_directory, os.path.basename(r2file)) # working_directory+'/'+os.path.basename(r2file)
flash_command = "{2} {0} {1}".format(r1file, r2file, flash_location)
parameters['o'] = outfile
parameters['d'] = working_directory
for p, val in parameters.iteritems():
flash_command += ' -{0} {1}'.format(p, str(val))
flash_command += ' -q' # run on quiet command
# os.system(flash_command)
worked = subprocess.call(flash_command, shell=True)
if worked > 0:
raise Exception('Flash failed')
os.rename(os.path.join(working_directory, outfile + '.extendedFrags.fastq'), os.path.join(working_directory, outfile))
try:
read_count_r1_file = useful.file_line_count(r1file)
except Exception as e:
read_count_r1_file = 1
print("Could not get number of lines in read file: " + str(e))
try:
read_count_flashed_file = useful.file_line_count(os.path.join(working_directory, outfile))
except Exception as e:
read_count_flashed_file = 1
print("Could not get number of lines in outfile read file: " + str(e))
resulting_counts = (
os.path.join(working_directory, outfile),
read_count_flashed_file / 4,
read_count_r1_file / 4,
float(100) * (read_count_flashed_file / float(read_count_r1_file))
)
return resulting_counts
def run_pear(r1file, r2file, working_directory, outfile='', parameters={}, suffix='', num_threads=1, memory='1G'):
r1_path = useful.get_parent_dir(r1file)
r2_path = useful.get_parent_dir(r2file)
if r1file.endswith('.gz'):
print("Unzipping R1 File..")
r1file = useful.gunzip_python(r1file)
if r2file.endswith('.gz'):
print("Unzipping R2 File..")
r2file = useful.gunzip_python(r2file)
working_directory = os.path.abspath(working_directory)
if r1_path != working_directory:
os.rename(r1file, os.path.join(working_directory, os.path.basename(r1file)))
if r2_path != working_directory:
os.rename(r2file, os.path.join(working_directory, os.path.basename(r2file)))
if outfile == '':
outfile = os.path.basename(r1file).split('.')
for p, subs in enumerate(outfile):
if '_R1' in subs:
r_pos = subs.index("_R1")
outfile[p] = subs[:r_pos]
break
elif '_R2' in subs:
r_pos = subs.index("_R2")
outfile[p] = subs[:r_pos]
break
outfile = '.'.join(outfile)
else:
outfile = os.path.basename(outfile)
outfile = outfile.replace('.fastq', '').replace('.fasta', '')
outfile = os.path.join(working_directory, outfile)
if os.path.isfile(os.path.join(working_directory, outfile)): # in resulting_files:
print('WARNING: FILE {0} ALREADY PRESENT IN FOLDER. FILE WILL BE OVERWRITTEN'.format(working_directory + '/' + outfile))
r1file = os.path.join(working_directory, os.path.basename(r1file))
r2file = os.path.join(working_directory, os.path.basename(r2file))
pear_command = "{2} -f {0} -r {1}".format(r1file, r2file, pear_location)
parameters['o'] = outfile
parameters['y'] = memory
parameters['j'] = num_threads
for p, val in parameters.iteritems():
pear_command += ' -{0} {1}'.format(p, str(val))
worked = subprocess.call(pear_command, shell=True)
if worked > 0:
raise Exception('Error in pear program')
try:
read_count_r1_file = useful.file_line_count(r1file)
except Exception as e:
read_count_r1_file = 1
print("Could not get number of lines in read file: " + str(e))
try:
read_count_flashed_file = useful.file_line_count(outfile + '.assembled.fastq')
except Exception as e:
read_count_flashed_file = 1
print("Could not get number of lines in outfile read file: " + str(e))
resulting_counts = (
outfile + '.assembled.fastq',
read_count_flashed_file / 4,
read_count_r1_file / 4,
float(100) * (read_count_flashed_file / float(read_count_r1_file))
)
return resulting_counts
def run_trimmomatic(files, output_directory=None, method='SE', phred=None, optional_parameters={}):
'''
Wrapper function for running trimmomatic program within python
Trimmomatic will remove low quality bases from the ends of NGS reads using an average quality score in a given window size
Parameters
----------
files : string or list of strings
List of input filenames (fastq or fastq.gz) for the MISEQ files. We either accept a single string or a list of two strings.
working_directory : string, default none
Pathname of desired output directory
outfile : string, default empty string
Desired filename name
method : SE or PE, default 'SE'
String representing whether to treat input files as single (SE) or paired-end files (PE)
phred : integer, default None
If None, then will rely on trimmomatic to guess the quality encoding. If a number, then will pass this value into the phred field.
optional_parameters : dict, default empty parameters
An optional dict of all parameters you would like to pass to trimmomatic
http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/TrimmomaticManual_V0.32.pdf
'''
method = method.upper()
if method not in ['SE', 'PE']:
raise Exception('Incorrect value provided for parameter "method". Provided value: ' + method)
if not isinstance(files, list):
files = [files]
if len(files) > 2:
raise Exception(str(len(files)) + 'total files have been passed to function. We only except 1 or 2 filepaths representing the R1/R2 reads')
for i, f in enumerate(files):
if f.endswith('.gz'):
print('Unzipping: ', f)
files[i] = useful.gunzip_python(f)
output_directory = useful.get_parent_dir(files[0]) if not output_directory else os.path.abspath(output_directory)
return_file_names = []
command_loops = []
if method == 'SE':
# Trim each file at a time
for f in files:
input_file_names = []
output_file_names = []
input_file_names.append('"' + f + '"')
out = f[:-6] if f.endswith('.fastq') else f
output_file_names.extend(['"' + out + '.trimmed.fastq"'])
return_file_names.append(out + '.trimmed.fastq')
command_loops.append([input_file_names, output_file_names])
else:
input_file_names = []
output_file_names = []
# trim all files simultaneously
for f in files:
input_file_names.append('"' + f + '"')
out = f[:-6] if f.endswith('.fastq') else f
output_file_names.extend(['"' + out + '.trimmed.fastq"', '"' + out + '.trimmed.unpaired.fastq"'])
return_file_names.append(out + '.trimmed.fastq')
command_loops.append([input_file_names, output_file_names])
phred_var = '-phred' + str(phred) if phred else ''
# We should change the java folder to recognize /usr/local/bin...
for loops in command_loops:
inputs = loops[0]
outputs = loops[1]
trim_command = 'java -jar {5} {0} {4} -threads 2 {1} {2} {3}'.format(method, ' '.join(inputs), ' '.join(outputs), ' '.join([key + ':' + str(value) for key, value in optional_parameters.iteritems()]), phred_var, trimmomatic_location)
worked = subprocess.call(trim_command, shell=True)
if worked > 0:
raise Exception('Trimmomatic failed')
return return_file_names
def run_pear(r1file,
r2file,
working_directory,
outfile='',
parameters={},
suffix='',
num_threads=1,
memory='1G'):
r1_path = useful.get_parent_dir(r1file)
r2_path = useful.get_parent_dir(r2file)
if r1file.endswith('.gz'):
print("Unzipping R1 File..")
r1file = useful.gunzip_python(r1file)
if r2file.endswith('.gz'):
print("Unzipping R2 File..")
r2file = useful.gunzip_python(r2file)
working_directory = os.path.abspath(working_directory)
if r1_path != working_directory:
os.rename(r1file,
os.path.join(working_directory, os.path.basename(r1file)))
if r2_path != working_directory:
os.rename(r2file,
os.path.join(working_directory, os.path.basename(r2file)))
if outfile == '':
outfile = os.path.basename(r1file).split('.')
for p, subs in enumerate(outfile):
if '_R1' in subs:
r_pos = subs.index("_R1")
outfile[p] = subs[:r_pos]
break
elif '_R2' in subs:
r_pos = subs.index("_R2")
outfile[p] = subs[:r_pos]
break
outfile = '.'.join(outfile)
else:
outfile = os.path.basename(outfile)
outfile = outfile.replace('.fastq', '').replace('.fasta', '')
outfile = os.path.join(working_directory, outfile)
if os.path.isfile(os.path.join(working_directory,
outfile)): # in resulting_files:
print(
'WARNING: FILE {0} ALREADY PRESENT IN FOLDER. FILE WILL BE OVERWRITTEN'
.format(working_directory + '/' + outfile))
r1file = os.path.join(working_directory, os.path.basename(r1file))
r2file = os.path.join(working_directory, os.path.basename(r2file))
pear_command = "{2} -f {0} -r {1}".format(r1file, r2file, pear_location)
parameters['o'] = outfile
parameters['y'] = memory
parameters['j'] = num_threads
for p, val in parameters.iteritems():
pear_command += ' -{0} {1}'.format(p, str(val))
worked = subprocess.call(pear_command, shell=True)
if worked > 0:
raise Exception('Error in pear program')
try:
read_count_r1_file = useful.file_line_count(r1file)
except Exception as e:
read_count_r1_file = 1
print("Could not get number of lines in read file: " + str(e))
try:
read_count_flashed_file = useful.file_line_count(outfile +
'.assembled.fastq')
except Exception as e:
read_count_flashed_file = 1
print("Could not get number of lines in outfile read file: " + str(e))
resulting_counts = (outfile + '.assembled.fastq',
read_count_flashed_file / 4, read_count_r1_file / 4,
float(100) *
(read_count_flashed_file / float(read_count_r1_file)))
return resulting_counts
def run_flash(r1file,
r2file,
working_directory,
outfile='',
parameters={},
suffix=''):
r1_path = useful.get_parent_dir(r1file) # '/'.join(r1file.split('/')[:-1])
r2_path = useful.get_parent_dir(r2file) # '/'.join(r2file.split('/')[:-1])
if not parameters:
print "PARAMETERS NOT PASSED INTO FLASH PROGRAM. USING DEFAULT IGSEQ PARAMETERS: R = 300, F = 400"
parameters = {'r': 300, 'f': 400}
if r1file.endswith('.gz'):
print "Unzipping R1 File.."
r1file = useful.gunzip_python(r1file)
if r2file.endswith('.gz'):
print "Unzipping R2 File.."
r2file = useful.gunzip_python(r2file)
working_directory = os.path.abspath(working_directory)
if r1_path != working_directory:
os.rename(r1file,
os.path.join(working_directory, os.path.basename(r1file)))
if r2_path != working_directory:
os.rename(r2file,
os.path.join(working_directory, os.path.basename(r2file)))
if outfile == '':
outfile = os.path.basename(r1file).split('.')
for p, subs in enumerate(outfile):
if '_R1' in subs:
r_pos = subs.index("_R1")
outfile[p] = subs[:r_pos]
break
elif '_R2' in subs:
r_pos = subs.index("_R2")
outfile[p] = subs[:r_pos]
break
outfile = '.'.join(outfile)
else:
outfile = os.path.basename(outfile)
outfile = outfile.replace('.fastq', '').replace('.fasta', '')
outfile += '.flashed' + suffix
if os.path.isfile(os.path.join(working_directory,
outfile)): # in resulting_files:
print(
'WARNING: FILE {0} ALREADY PRESENT IN FOLDER. FILE WILL BE OVERWRITTEN'
.format(working_directory + '/' + outfile))
r1file = os.path.join(working_directory, os.path.basename(
r1file)) # working_directory+'/'+os.path.basename(r1file)
r2file = os.path.join(working_directory, os.path.basename(
r2file)) # working_directory+'/'+os.path.basename(r2file)
flash_command = "{2} {0} {1}".format(r1file, r2file, flash_location)
parameters['o'] = outfile
parameters['d'] = working_directory
for p, val in parameters.iteritems():
flash_command += ' -{0} {1}'.format(p, str(val))
flash_command += ' -q' # run on quiet command
# os.system(flash_command)
worked = subprocess.call(flash_command, shell=True)
if worked > 0:
raise Exception('Flash failed')
os.rename(
os.path.join(working_directory, outfile + '.extendedFrags.fastq'),
os.path.join(working_directory, outfile))
try:
read_count_r1_file = useful.file_line_count(r1file)
except Exception as e:
read_count_r1_file = 1
print("Could not get number of lines in read file: " + str(e))
try:
read_count_flashed_file = useful.file_line_count(
os.path.join(working_directory, outfile))
except Exception as e:
read_count_flashed_file = 1
print("Could not get number of lines in outfile read file: " + str(e))
resulting_counts = (os.path.join(working_directory,
outfile), read_count_flashed_file / 4,
read_count_r1_file / 4, float(100) *
(read_count_flashed_file / float(read_count_r1_file)))
return resulting_counts
def run_trimmomatic(files,
output_directory=None,
method='SE',
phred=None,
optional_parameters={}):
'''
Wrapper function for running trimmomatic program within python
Trimmomatic will remove low quality bases from the ends of NGS reads using an average quality score in a given window size
Parameters
----------
files : string or list of strings
List of input filenames (fastq or fastq.gz) for the MISEQ files. We either accept a single string or a list of two strings.
working_directory : string, default none
Pathname of desired output directory
outfile : string, default empty string
Desired filename name
method : SE or PE, default 'SE'
String representing whether to treat input files as single (SE) or paired-end files (PE)
phred : integer, default None
If None, then will rely on trimmomatic to guess the quality encoding. If a number, then will pass this value into the phred field.
optional_parameters : dict, default empty parameters
An optional dict of all parameters you would like to pass to trimmomatic
http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/TrimmomaticManual_V0.32.pdf
'''
method = method.upper()
if method not in ['SE', 'PE']:
raise Exception(
'Incorrect value provided for parameter "method". Provided value: '
+ method)
if not isinstance(files, list):
files = [files]
if len(files) > 2:
raise Exception(
str(len(files)) +
'total files have been passed to function. We only except 1 or 2 filepaths representing the R1/R2 reads'
)
for i, f in enumerate(files):
if f.endswith('.gz'):
print('Unzipping: ', f)
files[i] = useful.gunzip_python(f)
output_directory = useful.get_parent_dir(
files[0]) if not output_directory else os.path.abspath(
output_directory)
return_file_names = []
command_loops = []
if method == 'SE':
# Trim each file at a time
for f in files:
input_file_names = []
output_file_names = []
input_file_names.append('"' + f + '"')
out = f[:-6] if f.endswith('.fastq') else f
output_file_names.extend(['"' + out + '.trimmed.fastq"'])
return_file_names.append(out + '.trimmed.fastq')
command_loops.append([input_file_names, output_file_names])
else:
input_file_names = []
output_file_names = []
# trim all files simultaneously
for f in files:
input_file_names.append('"' + f + '"')
out = f[:-6] if f.endswith('.fastq') else f
output_file_names.extend([
'"' + out + '.trimmed.fastq"',
'"' + out + '.trimmed.unpaired.fastq"'
])
return_file_names.append(out + '.trimmed.fastq')
command_loops.append([input_file_names, output_file_names])
phred_var = '-phred' + str(phred) if phred else ''
# We should change the java folder to recognize /usr/local/bin...
for loops in command_loops:
inputs = loops[0]
outputs = loops[1]
trim_command = 'java -jar {5} {0} {4} -threads 2 {1} {2} {3}'.format(
method, ' '.join(inputs), ' '.join(outputs), ' '.join([
key + ':' + str(value)
for key, value in optional_parameters.iteritems()
]), phred_var, trimmomatic_location)
worked = subprocess.call(trim_command, shell=True)
if worked > 0:
raise Exception('Trimmomatic failed')
return return_file_names