def writeResults(results, seq_file, out_args):
"""
Formats results and writes to output files
Arguments:
results = assembled results dictionary
seq_file = the sample sequence file name
out_args = common output argument dictionary from parseCommonArgs
Returns:
a tuple of (position error, nucleotide pairwise error, quality error, sequence set) file names
"""
pos_df = results['pos']
nuc_df = results['nuc']
qual_df = results['qual']
set_df = results['set']
# Type conversion to int of mismatch and total columns
pos_df[['mismatch', 'total']] = pos_df[['mismatch', 'total']].astype(int)
nuc_df[['mismatch', 'total']] = nuc_df[['mismatch', 'total']].astype(int)
qual_df[['mismatch', 'total']] = qual_df[['mismatch', 'total']].astype(int)
set_df[['mismatch', 'total']] = set_df[['mismatch', 'total']].astype(int)
# Write to tab delimited files
file_args = {'out_dir':out_args['out_dir'], 'out_name':out_args['out_name'], 'out_type':'tab'}
with getOutputHandle(seq_file, 'error-position', **file_args) as pos_handle, \
getOutputHandle(seq_file, 'error-quality', **file_args) as qual_handle, \
getOutputHandle(seq_file, 'error-nucleotide', **file_args) as nuc_handle, \
getOutputHandle(seq_file, 'error-set', **file_args) as set_handle:
pos_df.to_csv(pos_handle, sep='\t', na_rep='NA', index_label='POSITION',
cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f')
nuc_df.to_csv(nuc_handle, sep='\t', na_rep='NA', index_label=['OBSERVED', 'REFERENCE'],
cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f')
qual_df.to_csv(qual_handle, sep='\t', na_rep='NA', index_label='Q',
cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f')
set_df.to_csv(set_handle, sep='\t', na_rep='NA', index_label='SET_COUNT',
cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f')
return (pos_handle.name, qual_handle.name, nuc_handle.name, set_handle.name)
def _open(x, out_file=out_file):
if out_file is not None and x == 'pass':
handle = open(out_file, 'w')
else:
handle = getOutputHandle(seq_file,
'convert-%s' % x,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
return handle
def _open(x, in_file, out_name, out_file=out_file):
if out_file is not None and x == 'pass':
handle = open(out_file, 'w')
else:
handle = getOutputHandle(in_file,
out_label='%s-%s' % (label, x),
out_dir=out_args['out_dir'],
out_name=out_name,
out_type=out_type)
return handle
def addDbFile(db_file, fields, values, out_args=default_out_args):
"""
Adds field and value pairs to a database file
Arguments:
db_file = the database file name
fields = a list of fields to add
values = a list of values to assign to all rows of each field
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Define fields and values to append
add_dict = {k:v for k,v in zip(fields, values) if k not in db_iter.fieldnames}
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write updated row
rec.update(add_dict)
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args):
"""
Adds an index column to a database file
Arguments:
db_file = the database file name
field = the name of the index field to add
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'index'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-index',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=field)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Add count and write updated row
rec.update({field: rec_count})
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args):
"""
Deletes entire fields from a database file
Arguments:
db_file = the database file name
fields = a list of fields to drop
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-drop',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write row
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args):
"""
Adds an index column to a database file
Arguments:
db_file = the database file name
field = the name of the index field to add
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'index'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=field)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Add count and write updated row
rec.update({field:rec_count})
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def writeOffsetFile(primer_file, align_func=runMuscle, align_args={},
reverse=False, out_args=default_out_args):
"""
Generates an offset table from a sequence file
Arguments:
primer_file = name of file containing primer sequences
align_func = the function to use to align sequence sets
align_args = a dictionary of arguments to pass to align_func
reverse = if True count tail gaps; if False count head gaps
out_args = common output argument dictionary from parseCommonArgs
Returns:
the name of the offset output file
"""
log = OrderedDict()
log['START'] = 'AlignSets'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(primer_file)
log['REVERSE'] = reverse
printLog(log)
# Read primer file
primers = readPrimerFile(primer_file)
# Get offset dictionary
seq_list = [SeqRecord(Seq(v, IUPAC.ambiguous_dna), id=k) for k, v in primers.items()]
offset_dict = getOffsets(seq_list, align_func, align_args, reverse)
# Print log and write offsets to file
log = OrderedDict()
for s in seq_list:
log[s.id] = '%s %i' % (s.seq, offset_dict[s.id])
printLog(log)
# Write offset table
out_tag = 'reverse' if reverse else 'forward'
with getOutputHandle(primer_file, 'offsets-%s' % out_tag, out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab') as out_handle:
for k, v in offset_dict.items():
out_handle.write('%s\t%i\n' % (k, v))
# Print final log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['END'] = 'AlignSets'
printLog(log)
return out_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args):
"""
Deletes entire fields from a database file
Arguments:
db_file = the database file name
fields = a list of fields to drop
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write row
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args, cluster_func, cluster_args):
"""
Assembles results from a queue of individual sequence results and manages log/file I/O
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing continues
if False exit process
result_queue = a multiprocessing.Queue holding processQueue results
collect_queue = a multiprocessing.Queue to store collector return values
db_file = the input database file name
out_args = common output argument dictionary from parseCommonArgs
cluster_func = the function to call for carrying out clustering on distance matrix
cluster_args = a dictionary of arguments to pass to cluster_func
Returns:
None
(adds 'log' and 'out_files' to collect_dict)
"""
# Open output files
try:
# Iterate over Ig records to count and order by junction length
result_count = 0
records = {}
# print 'Reading file...'
db_iter = readDbFile(db_file)
for rec in db_iter:
records[rec.id] = rec
result_count += 1
records = OrderedDict(sorted(list(records.items()), key=lambda i: i[1].junction_length))
# Define empty matrix to store assembled results
dist_mat = np.zeros((result_count,result_count))
# Count records and define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined successful output handle
pass_handle = getOutputHandle(db_file,
out_label='clone-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE')
# Defined failed cloning output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
out_label='clone-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
fail_writer = None
# Open log file
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
row_count = rec_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
if row_count == 0:
print('PROGRESS> Assigning clones')
printProgress(row_count, result_count, 0.05, start_time)
# Update counts for iteration
row_count += 1
rec_count += len(result)
# Add result row to distance matrix
if result:
dist_mat[list(range(result_count-len(result),result_count)),result_count-len(result)] = result.results
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Calculate linkage and carry out clustering
# print dist_mat
clusters = cluster_func(dist_mat, **cluster_args) if dist_mat is not None else None
clones = {}
# print clusters
for i, c in enumerate(clusters):
clones.setdefault(c, []).append(records[list(records.keys())[i]])
# Write passed and failed records
clone_count = pass_count = fail_count = 0
if clones:
for clone in clones.values():
clone_count += 1
for i, rec in enumerate(clone):
rec.annotations['CLONE'] = clone_count
pass_writer.writerow(rec.toDict())
pass_count += 1
#result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction)
else:
for i, rec in enumerate(result.data):
fail_writer.writerow(rec.toDict())
fail_count += 1
#result.log['CLONE0-%i' % (i + 1)] = str(rec.junction)
# Print final progress
printProgress(row_count, result_count, 0.05, start_time)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
# Update return list
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
def writeResults(results, seq_file, out_args):
"""
Formats results and writes to output files
Arguments:
results : assembled results dictionary.
seq_file : the sample sequence file name.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
tuple: (position error, nucleotide pairwise error, quality error, sequence set) file names
"""
pos_df = results['pos']
nuc_df = results['nuc']
qual_df = results['qual']
set_df = results['set']
dist_df = results['dist']
thresh_df = results['thresh']
# Type conversion to int of mismatch and total columns
pos_df[['mismatch', 'total']] = pos_df[['mismatch', 'total']].astype(int)
nuc_df[['mismatch', 'total']] = nuc_df[['mismatch', 'total']].astype(int)
qual_df[['mismatch', 'total']] = qual_df[['mismatch', 'total']].astype(int)
set_df[['mismatch', 'total']] = set_df[['mismatch', 'total']].astype(int)
dist_df[['all']] = dist_df[['all']].astype(int)
# Write to tab delimited files
file_args = {
'out_dir': out_args['out_dir'],
'out_name': out_args['out_name'],
'out_type': 'tab'
}
with getOutputHandle(seq_file, 'error-position', **file_args) as pos_handle, \
getOutputHandle(seq_file, 'error-quality', **file_args) as qual_handle, \
getOutputHandle(seq_file, 'error-nucleotide', **file_args) as nuc_handle, \
getOutputHandle(seq_file, 'error-set', **file_args) as set_handle, \
getOutputHandle(seq_file, 'distance-set', **file_args) as dist_handle, \
getOutputHandle(seq_file, 'threshold-set', **file_args) as thresh_handle:
pos_df.to_csv(pos_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='POSITION',
columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=[
'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR',
'EMPIRICAL_Q'
],
float_format='%.6f')
nuc_df.to_csv(nuc_handle,
sep='\t',
na_rep='NA',
index=True,
index_label=['REFERENCE', 'OBSERVED'],
columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=[
'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR',
'EMPIRICAL_Q'
],
float_format='%.6f')
qual_df.to_csv(
qual_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='Q',
columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=[
'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR',
'EMPIRICAL_Q'
],
float_format='%.6f')
set_df.to_csv(set_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='SET_COUNT',
columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'],
header=[
'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR',
'EMPIRICAL_Q'
],
float_format='%.6f')
dist_df.to_csv(dist_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='DISTANCE',
columns=['all'],
header=['ALL'],
float_format='%.6f')
thresh_df.to_csv(thresh_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='TYPE',
columns=['thresh'],
header=['THRESHOLD'],
float_format='%.6f')
return (pos_handle.name, qual_handle.name, nuc_handle.name,
set_handle.name, dist_handle.name, thresh_handle.name)
def convertDbFasta(db_file,
id_field=default_id_field,
seq_field=default_seq_field,
meta_fields=None,
out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sequences
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
out_type = 'fasta'
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='sequences',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Get SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields,
out_args['delimiter'])
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, out_type)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def assembleCloneGermline(db_file,
repo,
seq_field=default_seq_field,
v_field=default_v_field,
germ_types=default_germ_types,
out_args=default_out_args):
"""
Assemble one germline sequence for each clone in a tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types,
str) else ','.join(germ_types)
log['CLONED'] = 'True'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
references = readRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS']
add_fields += ['GERMLINE_V_CALL']
add_fields += ['GERMLINE_D_CALL']
add_fields += ['GERMLINE_J_CALL']
# Create output file handle and Db writer
writers = {}
pass_handle = getOutputHandle(db_file,
'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
writers['fail'] = getDbWriter(fail_handle,
db_file,
add_fields=add_fields)
else:
fail_handle = None
writers['fail'] = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
counts = {}
clone_count = counts['pass'] = counts['fail'] = 0
# Iterate over rows
clone = 'initial'
clone_dict = OrderedDict()
for i, row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
# Clone isn't over yet
if row.get('CLONE', '') == clone:
clone_dict[i] = row
# Clone just finished
elif clone_dict:
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, references,
germ_types, v_field, seq_field,
counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Now deal with current row (first of next clone)
clone = row['CLONE']
clone_dict = OrderedDict([(i, row)])
# Last case is only for first row of file
else:
clone = row['CLONE']
clone_dict = OrderedDict([(i, row)])
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, references, germ_types,
v_field, seq_field, counts, writers,
out_args)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i + 1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = counts['pass']
log['FAIL'] = counts['fail']
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def sortDbFile(db_file, field, numeric=False, descend=False,
out_args=default_out_args):
"""
Sorts records by values in an annotation field
Arguments:
db_file = the database filename
field = the field name to sort by
numeric = if True sort field numerically;
if False sort field alphabetically
descend = if True sort in descending order;
if False sort in ascending order
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Store all records in a dictionary
start_time = time()
printMessage("Indexing: Running", start_time=start_time)
db_dict = {i:r for i, r in enumerate(db_iter)}
result_count = len(db_dict)
# Sort db_dict by field values
tag_dict = {k:v[field] for k, v in db_dict.items()}
if numeric: tag_dict = {k:float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend)
printMessage("Indexing: Done", start_time=start_time, end=True)
# Iterate over records
start_time = time()
rec_count = 0
for key in sorted_keys:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write records
pass_writer.writerow(db_dict[key])
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectDbQueue(alive, result_queue, collect_queue, db_file, task_label, out_args,
add_fields=None):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive : multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue : multiprocessing.Queue holding worker results
collect_queue : multiprocessing.Queue to store collector return values
db_file : Database file name
task_label : Task label used to tag the output files
out_args : Common output argument dictionary from parseCommonArgs
add_fields : List of fields added to the writer not present in the in_file;
if None do not add fields
Returns:
None : Adds a dictionary with key value pairs to collect_queue containing
'log' defining a log object,
'out_files' defining the output file names
"""
try:
result_count = countDbFile(db_file)
# Define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined valid alignment output handle
pass_handle = getOutputHandle(db_file,
'%s-pass' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
# Defined failed alignment output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'%s-fail' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
set_count = rec_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(pass_count, result_count, 0.05, start_time)
# Update counts for current iteration
set_count += 1
rec_count += result.data_count
# Write log
printLog(result.log, handle=log_handle)
# Write alignments
if result:
pass_count += result.data_count
for rec in result.results:
pass_writer.writerow(rec.toDict())
else:
fail_count += result.data_count
if fail_handle is not None:
for rec in result.data:
pass_writer.writerow(rec.toDict())
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(pass_count, result_count, 0.05, start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GROUPS'] = set_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
except:
alive.value = False
raise
return None
def collectDbQueue(alive,
result_queue,
collect_queue,
db_file,
task_label,
out_args,
add_fields=None):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive : multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue : multiprocessing.Queue holding worker results
collect_queue : multiprocessing.Queue to store collector return values
db_file : Database file name
task_label : Task label used to tag the output files
out_args : Common output argument dictionary from parseCommonArgs
add_fields : List of fields added to the writer not present in the in_file;
if None do not add fields
Returns:
None : Adds a dictionary with key value pairs to collect_queue containing
'log' defining a log object,
'out_files' defining the output file names
"""
try:
result_count = countDbFile(db_file)
# Define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined valid alignment output handle
pass_handle = getOutputHandle(db_file,
'%s-pass' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
# Defined failed alignment output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'%s-fail' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
set_count = rec_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(pass_count, result_count, 0.05, start_time)
# Update counts for current iteration
set_count += 1
rec_count += result.data_count
# Write log
if result.log is not None:
printLog(result.log, handle=log_handle)
# Write alignments
if result:
pass_count += result.data_count
if isinstance(result.results, IgRecord):
pass_writer.writerow(result.results.toDict())
else:
for rec in result.results:
pass_writer.writerow(rec.toDict())
else:
fail_count += result.data_count
if fail_handle is not None:
if isinstance(result.data, IgRecord):
pass_writer.writerow(result.data.toDict())
else:
for rec in result.data:
fail_writer.writerow(rec.toDict())
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(pass_count, result_count, 0.05, start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GROUPS'] = set_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log': log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
except:
alive.value = False
raise
return None
def writeOffsetFile(primer_file, align_func=runMuscle, align_args={},
reverse=False, out_file=None, out_args=default_out_args):
"""
Generates an offset table from a sequence file
Arguments:
primer_file : name of file containing primer sequences.
align_func : the function to use to align sequence sets.
align_args : a dictionary of arguments to pass to align_func.
reverse : if True count tail gaps; if False count head gaps.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: the name of the offset output file.
"""
log = OrderedDict()
log['START'] = 'AlignSets'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(primer_file)
log['REVERSE'] = reverse
printLog(log)
# Read primer file
primers = readPrimerFile(primer_file)
# Get offset dictionary
seq_list = [SeqRecord(Seq(v, IUPAC.ambiguous_dna), id=k) for k, v in primers.items()]
offset_dict = getOffsets(seq_list, align_func, align_args, reverse)
# Print log and write offsets to file
log = OrderedDict()
for s in seq_list:
log[s.id] = '%s %i' % (s.seq, offset_dict[s.id])
printLog(log)
# Open output handle
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_tag = 'reverse' if reverse else 'forward'
out_handle = getOutputHandle(primer_file,
'offsets-%s' % out_tag,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Write offset table
for k, v in offset_dict.items():
out_handle.write('%s\t%i\n' % (k, v))
# Print final log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['END'] = 'AlignSets'
printLog(log)
# Close handle
out_handle.close()
return out_handle.name
def collapseSeq(seq_file, max_missing=default_max_missing, uniq_fields=None,
copy_fields=None, copy_actions=None, max_field=None, min_field=None,
inner=False, keep_missing=False, out_args=default_out_args):
"""
Removes duplicate sequences from a file
Arguments:
seq_file = filename of the sequence file to sample from
max_missing = number of ambiguous characters to allow in a unique sequence
uniq_fields = a list of annotations that define a sequence as unique if they differ
copy_fields = a list of annotations to copy into unique sequence annotations
copy_actions = the list of collapseAnnotation actions to take on copy_fields
max_field = a numeric field whose maximum value determines the retained sequence
min_field = a numeric field whose minimum value determines the retained sequence
inner = if True exclude consecutive outer ambiguous characters from iterations and matching
keep_missing = if True retain sequences with more ambiguous characters than max_missing as unique
out_args = common output argument dictionary from parseCommonArgs
Returns:
the collapsed output file name
"""
log = OrderedDict()
log['START'] = 'CollapseSeq'
log['FILE'] = os.path.basename(seq_file)
log['MAX_MISSING'] = max_missing
log['UNIQ_FIELDS'] = ','.join([str(x) for x in uniq_fields]) \
if uniq_fields is not None else None
log['COPY_FIELDS'] = ','.join([str(x) for x in copy_fields]) \
if copy_fields is not None else None
log['COPY_ACTIONS'] = ','.join([str(x) for x in copy_actions]) \
if copy_actions is not None else None
log['MAX_FIELD'] = max_field
log['MIN_FIELD'] = min_field
log['INNER'] = inner
log['KEEP_MISSING'] = keep_missing
printLog(log)
# TODO: storing all sequences in memory is faster
# Read input file
in_type = getFileType(seq_file)
#seq_dict = readSeqFile(seq_file, index=True)
seq_dict = SeqIO.to_dict(readSeqFile(seq_file, index=False))
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Count total sequences
rec_count = len(seq_dict)
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
# Find sequences with duplicates
uniq_dict = {}
# Added list typing for compatibility issue with Python 2.7.5 on OS X
# TypeError: object of type 'dictionary-keyiterator' has no len()
search_keys = list(seq_dict.keys())
dup_keys = []
for n in range(0, max_missing + 1):
# Find unique sequences
uniq_dict, search_keys, dup_list = findUniqueSeq(uniq_dict, search_keys, seq_dict, n,
uniq_fields, copy_fields,
max_field, min_field, inner,
out_args['delimiter'])
# Update list of duplicates
dup_keys.extend(dup_list)
# Update log
log = OrderedDict()
log['ITERATION'] = n + 1
log['MISSING'] = n
log['UNIQUE'] = len(uniq_dict)
log['DUPLICATE'] = len(dup_keys)
log['UNDETERMINED'] = len(search_keys)
printLog(log, handle=log_handle)
# Break if no keys to search remain
if len(search_keys) == 0: break
# Write unique sequences
with getOutputHandle(seq_file, 'collapse-unique', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type']) \
as uniq_handle:
for val in uniq_dict.values():
# Define output sequence
out_seq = val[0]
out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter'])
out_app = OrderedDict()
if copy_fields is not None and copy_actions is not None:
for f, a, s in zip(copy_fields, copy_actions, val[3:]):
out_app[f] = s
out_app = collapseAnnotation(out_app, a, f, delimiter=out_args['delimiter'])
out_ann.pop(f, None)
out_app['DUPCOUNT'] = val[1]
out_ann = mergeAnnotation(out_ann, out_app, delimiter=out_args['delimiter'])
out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter'])
out_seq.description = ''
# Write unique sequence
SeqIO.write(out_seq, uniq_handle, out_args['out_type'])
# Write sequence with high missing character counts
if keep_missing:
for k in search_keys:
out_seq = seq_dict[k]
out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter'])
out_ann = mergeAnnotation(out_ann, {'DUPCOUNT':1}, delimiter=out_args['delimiter'])
out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter'])
out_seq.description = ''
SeqIO.write(out_seq, uniq_handle, out_args['out_type'])
# Write sequence with high missing character counts
if out_args['failed'] and not keep_missing:
with getOutputHandle(seq_file, 'collapse-undetermined', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type']) \
as missing_handle:
for k in search_keys:
SeqIO.write(seq_dict[k], missing_handle, out_args['out_type'])
if out_args['failed']:
# Write duplicate sequences
with getOutputHandle(seq_file, 'collapse-duplicate', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type']) \
as dup_handle:
for k in dup_keys:
SeqIO.write(seq_dict[k], dup_handle, out_args['out_type'])
# Print log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(uniq_handle.name)
log['SEQUENCES'] = rec_count
log['UNIQUE'] = len(uniq_dict)
log['DUPLICATE'] = len(dup_keys)
log['UNDETERMINED'] = len(search_keys)
log['END'] = 'CollapseSeq'
printLog(log)
# Close file handles
if log_handle is not None: log_handle.close()
return uniq_handle.name
def assembleCloneGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args):
"""
Assemble one germline sequence for each clone in a tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types)
log['CLONED'] = 'True'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
repo_dict = getRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
# Create output file handle and Db writer
writers = {}
pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
writers['fail'] = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
counts = {}
clone_count = counts['pass'] = counts['fail'] = 0
# Iterate over rows
clone = 'initial'
clone_dict = OrderedDict()
for i,row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
# Clone isn't over yet
if row.get('CLONE','') == clone:
clone_dict[row["SEQUENCE_ID"]] = row
# Clone just finished
elif clone_dict:
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types,
v_field, seq_field, counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Now deal with current row (first of next clone)
clone = row['CLONE']
clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)])
# Last case is only for first row of file
else:
clone = row['CLONE']
clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)])
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field,
seq_field, counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i+1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = counts['pass']
log['FAIL'] = counts['fail']
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args):
"""
Write germline sequences to tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types)
log['CLONED'] = 'False'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
repo_dict = getRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
# Create output file handle and Db writer
pass_handle = getOutputHandle(db_file, 'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file, 'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
fail_writer = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
pass_count = fail_count = 0
# Iterate over rows
for i,row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
result_log, germlines = joinGermline(row, repo_dict, germ_types, v_field, seq_field)
# Add germline field(s) to dictionary
if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full']
if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask']
if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly']
# Write row to pass or fail file
if 'ERROR' in result_log:
fail_count += 1
if fail_writer is not None: fail_writer.writerow(row)
else:
result_log['SEQUENCE'] = row[seq_field]
result_log['GERMLINE'] = germlines['full']
result_log['REGIONS'] = germlines['regions']
pass_count += 1
pass_writer.writerow(row)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i+1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def estimateBarcode(seq_file,
barcode_field=default_barcode_field,
distance_types=default_distance_types,
out_args=default_out_args):
"""
Calculates error rates of barcode sequences
Arguments:
seq_file : the sample sequence file name
barcode_field : the annotation field containing barcode sequences.
distance_types : distance types to include.
out_args : common output argument dictionary from parseCommonArgs
Returns:
tuple: names of the output files.
"""
# Function to extract to make SeqRecord object from a barcode annotation
def _barcode(seq, field=barcode_field, delimiter=out_args['delimiter']):
header = parseAnnotation(seq.description, delimiter=delimiter)
return header[field]
# Print parameter info
log = OrderedDict()
log['START'] = 'EstimateError'
log['COMMAND'] = 'barcode'
log['FILE'] = os.path.basename(seq_file)
log['BARCODE_FIELD'] = barcode_field
printLog(log)
# Count sequence file and parse into a list of SeqRecords
result_count = countSeqFile(seq_file)
barcode_iter = (_barcode(x) for x in readSeqFile(seq_file))
# Compute bin_count defaults to the length of the barcode + 1
bin_count = len(_barcode(next(readSeqFile(seq_file)))) + 1
mismatch = initializeMismatchDictionary(0,
distance_types=distance_types,
bin_count=bin_count)
# Calculate distances
distance_mismatch = calculateDistances(barcode_iter, bin_count=bin_count)
mismatch['dist'] = {
header: distance_mismatch[header]
for header in distance_types
}
# Generate a df
dist_df = pd.DataFrame.from_dict(mismatch['dist'])
dist_df.index = dist_df.index / len(dist_df.index)
dist_df[['all']] = dist_df[['all']].astype(int)
#find the threshold (average minimum between 0 and 0.75)
dist = mismatch['dist']['all']
thresh_df = pd.DataFrame.from_dict({'thresh': {'ALL': dist_df.index[int(np.mean([index for index in np.argsort(dist[:int(len(dist)*0.75)]) \
if dist[index] == np.min(dist)]))]}
})
file_args = {
'out_dir': out_args['out_dir'],
'out_name': out_args['out_name'],
'out_type': 'tab'
}
# Output as tsv
with getOutputHandle(seq_file, 'distance-barcode', **file_args) as dist_handle, \
getOutputHandle(seq_file, 'threshold-barcode', **file_args) as thresh_handle:
dist_df.to_csv(dist_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='DISTANCE',
columns=['all'],
header=['ALL'],
float_format='%.6f')
thresh_df.to_csv(thresh_handle,
sep='\t',
na_rep='NA',
index=True,
index_label='TYPE',
columns=['thresh'],
header=['THRESHOLD'],
float_format='%.6f')
# Update log
log['OUTPUT1'] = os.path.basename(dist_handle.name)
log['OUTPUT2'] = os.path.basename(thresh_handle.name)
log['SEQUENCES'] = result_count
log['ALL_THRESHOLD'] = '%.6f' % thresh_df['thresh']['ALL']
log['END'] = 'EstimateError'
printLog(log)
return (dist_handle.name, thresh_handle.name)
def convertDbClip(db_file, id_field=default_id_field, seq_field=default_seq_field,
germ_field=default_germ_field, cluster_field=None,
meta_fields=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sample sequences
germ_field = the field containing germline sequences
cluster_field = the field containing clonal groupings
if None write the germline for each record
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
log['GERM_FIELD'] = germ_field
log['CLUSTER_FIELD'] = cluster_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='clip')
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = germ_count = pass_count = fail_count = 0
cluster_last = None
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Update cluster ID
cluster = rec.get(cluster_field, None)
# Get germline SeqRecord when needed
if cluster_field is None:
germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
elif cluster != cluster_last:
germ = getDbSeqRecord(rec, cluster_field, germ_field,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
else:
germ = None
# Get read SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields,
delimiter=out_args['delimiter'])
# Write germline
if germ is not None:
germ_count += 1
SeqIO.write(germ, pass_handle, 'fasta')
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, 'fasta')
else:
fail_count += 1
# Set last cluster ID
cluster_last = cluster
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GERMLINES'] = germ_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectPairQueue(alive, result_queue, collect_queue, result_count,
seq_file_1, seq_file_2, out_args):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue = a multiprocessing.Queue holding worker results
collect_queue = a multiprocessing.Queue holding collector return values
result_count = the number of expected assembled sequences
seq_file_1 = the first sequence file name
seq_file_2 = the second sequence file name
out_args = common output argument dictionary from parseCommonArgs
Returns:
None
(adds a dictionary of {log: log object, out_files: output file names} to collect_queue)
"""
try:
# Count records and define output format
out_type = getFileType(seq_file_1) if out_args['out_type'] is None \
else out_args['out_type']
# Defined valid assembly output handle
pass_handle = getOutputHandle(seq_file_1,
'assemble-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
# Defined failed assembly output handles
if out_args['failed']:
# Define output name
if out_args['out_name'] is None:
out_name_1 = out_name_2 = None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
fail_handle_1 = getOutputHandle(seq_file_1,
'assemble-fail',
out_dir=out_args['out_dir'],
out_name=out_name_1,
out_type=out_type)
fail_handle_2 = getOutputHandle(seq_file_2,
'assemble-fail',
out_dir=out_args['out_dir'],
out_name=out_name_2,
out_type=out_type)
else:
fail_handle_1 = None
fail_handle_2 = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
iter_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(iter_count, result_count, 0.05, start_time)
# Update counts for iteration
iter_count += 1
# Write log
printLog(result.log, handle=log_handle)
# Write assembled sequences
if result:
pass_count += 1
SeqIO.write(result.results, pass_handle, out_type)
else:
fail_count += 1
if fail_handle_1 is not None and fail_handle_2 is not None:
SeqIO.write(result.data[0], fail_handle_1, out_type)
SeqIO.write(result.data[1], fail_handle_2, out_type)
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(iter_count, result_count, 0.05, start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['PAIRS'] = iter_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
# Close file handles
pass_handle.close()
if fail_handle_1 is not None: fail_handle_1.close()
if fail_handle_2 is not None: fail_handle_2.close()
if log_handle is not None: log_handle.close()
except:
alive.value = False
raise
return None
def tableHeaders(seq_file, fields, out_file=None, out_args=default_out_args):
"""
Builds a table of sequence header annotations
Arguments:
seq_file : the sequence file name.
fields : the list of fields to output.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: output table file name
"""
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open file handles
seq_iter = readSeqFile(seq_file)
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(seq_file,
'headers',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Count records
result_count = countSeqFile(seq_file)
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time=start_time)
# Get annotations
seq_count += 1
ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter'])
# Write records
if ann:
pass_count += 1
out_writer.writerow(ann)
else:
fail_count += 1
# Print counts
printProgress(seq_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def selectDbFile(db_file, fields, values, logic='any', regex=False,
out_args=default_out_args):
"""
Selects records from a database file
Arguments:
db_file = the database file name
fields = a list of fields to check for selection criteria
values = a list of values defining selection targets
logic = one of 'any' or 'all' defining whether one or all fields must have a match.
regex = if False do exact full string matches; if True allow partial regex matches.
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define string match function
if regex:
def _match_func(x, patterns): return any([re.search(p, x) for p in patterns])
else:
def _match_func(x, patterns): return x in patterns
# Define logic function
if logic == 'any':
_logic_func = any
elif logic == 'all':
_logic_func = all
# Print console log
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'select'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
log['REGEX'] =regex
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Check for selection values in all fields
select = _logic_func([_match_func(rec.get(f, False), values) for f in fields])
# Write sequences
if select:
pass_count += 1
pass_writer.writerow(rec)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['SELECTED'] = pass_count
log['DISCARDED'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def modifyHeaders(seq_file, modify_func, modify_args, out_file=None, out_args=default_out_args):
"""
Modifies sequence headers
Arguments:
seq_file : the sequence file name.
modify_func : the function defining the modification operation.
modify_args : a dictionary of arguments to pass to modify_func.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: output file name.
"""
# Define subcommand label dictionary
cmd_dict = {addHeader: 'add',
copyHeader: 'copy',
collapseHeader: 'collapse',
deleteHeader: 'delete',
expandHeader: 'expand',
renameHeader: 'rename'}
# Print parameter info
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__)
log['FILE'] = os.path.basename(seq_file)
for k in sorted(modify_args):
v = modify_args[k]
log[k.upper()] = ','.join(v) if isinstance(v, list) else v
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(seq_file,
'reheader',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Count records
result_count = countSeqFile(seq_file)
# Iterate over sequences
start_time = time()
seq_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time=start_time)
#Update counts
seq_count += 1
# Modify header
header = parseAnnotation(seq.description, delimiter=out_args['delimiter'])
header = modify_func(header, delimiter=out_args['delimiter'], **modify_args)
# Write new sequence
seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, out_handle, out_args['out_type'])
# print counts
printProgress(seq_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def clusterBarcodes(seq_file,
ident=default_cluster_ident,
length_ratio=default_length_ratio,
barcode_field=default_barcode_field,
cluster_field=default_cluster_field,
cluster_prefix=default_cluster_prefix,
cluster_tool=default_cluster_tool,
cluster_exec=default_cluster_exec,
out_file=None,
out_args=default_out_args,
nproc=None):
"""
Performs clustering on sets of sequences
Arguments:
seq_file : the sample sequence file name.
ident : the identity threshold for clustering sequences.
length_ratio : minimum short/long length ratio allowed within a cluster.
barcode_field : the annotation field containing barcode sequences.
cluster_field : the name of the output cluster field.
cluster_prefix : string defining a prefix for the cluster identifier.
seq_start : the start position to trim sequences at before clustering.
seq_end : the end position to trim sequences at before clustering.
cluster_tool : the clustering tool to use; one of cd-hit or usearch.
cluster_exec : the path to the executable for usearch.
out_file : output file name. Automatically generated from the input file if None.
out_args : output arguments.
nproc : the number of processQueue processes;
if None defaults to the number of CPUs.
Returns:
str: the clustered output file name
"""
# Function to modify SeqRecord header with cluster identifier
def _header(seq,
cluster,
field=cluster_field,
prefix=cluster_prefix,
delimiter=out_args['delimiter']):
label = '%s%i' % (prefix, cluster)
header = parseAnnotation(seq.description, delimiter=delimiter)
header = mergeAnnotation(header, {field: label}, delimiter=delimiter)
seq.id = seq.name = flattenAnnotation(header, delimiter=delimiter)
seq.description = ''
return seq
# Function to extract to make SeqRecord object from a barcode annotation
def _barcode(seq, field=barcode_field, delimiter=out_args['delimiter']):
header = parseAnnotation(seq.description, delimiter=delimiter)
return SeqRecord(Seq(header[field]), id=seq.id)
# Print parameter info
log = OrderedDict()
log['START'] = 'ClusterSets'
log['COMMAND'] = 'barcode'
log['FILE'] = os.path.basename(seq_file)
log['IDENTITY'] = ident
log['BARCODE_FIELD'] = barcode_field
log['CLUSTER_FIELD'] = cluster_field
log['CLUSTER_PREFIX'] = cluster_prefix
log['CLUSTER_TOOL'] = cluster_tool
log['NPROC'] = nproc
printLog(log)
# Set cluster tool
try:
cluster_func = map_cluster_tool.get(cluster_tool)
except:
printError('Invalid clustering tool %s.' % cluster_tool)
# Check the minimum identity
if ident < min_cluster_ident[cluster_tool]:
printError('Minimum identity %s too low for clustering tool %s.' %
(str(ident), cluster_tool))
# Count sequence file and parse into a list of SeqRecords
result_count = countSeqFile(seq_file)
barcode_iter = (_barcode(x) for x in readSeqFile(seq_file))
# Perform clustering
start_time = time()
printMessage('Running %s' % cluster_tool, start_time=start_time, width=25)
cluster_dict = cluster_func(barcode_iter,
ident=ident,
length_ratio=length_ratio,
seq_start=0,
seq_end=None,
threads=nproc,
cluster_exec=cluster_exec)
printMessage('Done', start_time=start_time, end=True, width=25)
# Determine file type
if out_args['out_type'] is None:
out_args['out_type'] = getFileType(seq_file)
# Open output file handles
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(seq_file,
'cluster-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Open indexed sequence file
seq_dict = readSeqFile(seq_file, index=True)
# Iterate over sequence records and update header with cluster annotation
start_time = time()
rec_count = pass_count = 0
for cluster, id_list in cluster_dict.items():
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += len(id_list)
# TODO: make a generator. Figure out how to get pass_count updated
# Define output sequences
seq_output = [_header(seq_dict[x], cluster) for x in id_list]
# Write output
pass_count += len(seq_output)
SeqIO.write(seq_output, pass_handle, out_args['out_type'])
# Update progress
printProgress(rec_count, result_count, 0.05, start_time=start_time)
# Print log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLUSTERS'] = len(cluster_dict)
log['SEQUENCES'] = result_count
log['PASS'] = pass_count
log['FAIL'] = rec_count - pass_count
log['END'] = 'ClusterSets'
printLog(log)
# Close handles
pass_handle.close()
return pass_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args):
"""
Renames fields in a database file
Arguments:
db_file = the database file name
fields = a list of fields to rename
values = a list of new names for fields
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'rename'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['NAMES'] = ','.join(names)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-rename',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Get header and rename fields
header = (readDbFile(db_file, ig=False)).fieldnames
for f, n in zip(fields, names):
i = header.index(f)
header[i] = n
# Open writer and write new header
# TODO: should modify getDbWriter to take a list of fields
pass_writer = csv.DictWriter(pass_handle,
fieldnames=header,
dialect='excel-tab')
pass_writer.writeheader()
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore
# Rename fields
for f, n in zip(fields, names):
rec[n] = rec.pop(f)
# Write
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def convertDbBaseline(db_file,
id_field=default_id_field,
seq_field=default_seq_field,
germ_field=default_germ_field,
cluster_field=None,
meta_fields=None,
out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sample sequences
germ_field = the field containing germline sequences
cluster_field = the field containing clonal groupings
if None write the germline for each record
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
log['GERM_FIELD'] = germ_field
log['CLUSTER_FIELD'] = cluster_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='sequences',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='clip')
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = germ_count = pass_count = fail_count = 0
cluster_last = None
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Update cluster ID
cluster = rec.get(cluster_field, None)
# Get germline SeqRecord when needed
if cluster_field is None:
germ = getDbSeqRecord(rec,
id_field,
germ_field,
meta_fields,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
elif cluster != cluster_last:
germ = getDbSeqRecord(rec,
cluster_field,
germ_field,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
else:
germ = None
# Get read SeqRecord
seq = getDbSeqRecord(rec,
id_field,
seq_field,
meta_fields,
delimiter=out_args['delimiter'])
# Write germline
if germ is not None:
germ_count += 1
SeqIO.write(germ, pass_handle, 'fasta')
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, 'fasta')
else:
fail_count += 1
# Set last cluster ID
cluster_last = cluster
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GERMLINES'] = germ_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args):
"""
Divides a tab-delimited database file into segments by description tags
Arguments:
db_file = filename of the tab-delimited database file to split
field = the field name by which to split db_file
num_split = the numerical threshold by which to group sequences;
if None treat field as textual
out_args = common output argument dictionary from parseCommonArgs
Returns:
a list of output file names
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'split'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUM_SPLIT'] = num_split
printLog(log)
# Open IgRecord reader iter object
reader = readDbFile(db_file, ig=False)
# Determine total numbers of records
rec_count = countDbFile(db_file)
start_time = time()
count = 0
# Sort records into files based on textual field
if num_split is None:
# Create set of unique field tags
tmp_iter = readDbFile(db_file, ig=False)
tag_list = list(set([row[field] for row in tmp_iter]))
# Forbidden characters in filename and replacements
noGood = {
'\/': 'f',
'\\': 'b',
'?': 'q',
'\%': 'p',
'*': 's',
':': 'c',
'\|': 'pi',
'\"': 'dq',
'\'': 'sq',
'<': 'gt',
'>': 'lt',
' ': '_'
}
# Replace forbidden characters in tag_list
tag_dict = {}
for tag in tag_list:
for c, r in noGood.items():
tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \
if c in tag else tag_dict.get(tag, tag))
# Create output handles
handles_dict = {
tag: getOutputHandle(db_file,
'%s-%s' % (field, label),
out_type=out_args['out_type'],
out_name=out_args['out_name'],
out_dir=out_args['out_dir'])
for tag, label in tag_dict.items()
}
# Create Db writer instances
writers_dict = {
tag: getDbWriter(handles_dict[tag], db_file)
for tag in tag_dict
}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
# Write row to appropriate file
tag = row[field]
writers_dict[tag].writerow(row)
# Sort records into files based on numeric num_split
else:
num_split = float(num_split)
# Create output handles
handles_dict = {
'under':
getOutputHandle(db_file,
'under-%.1f' % num_split,
out_type=out_args['out_type'],
out_name=out_args['out_name'],
out_dir=out_args['out_dir']),
'atleast':
getOutputHandle(db_file,
'atleast-%.1f' % num_split,
out_type=out_args['out_type'],
out_name=out_args['out_name'],
out_dir=out_args['out_dir'])
}
# Create Db writer instances
writers_dict = {
'under': getDbWriter(handles_dict['under'], db_file),
'atleast': getDbWriter(handles_dict['atleast'], db_file)
}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
tag = row[field]
tag = 'under' if float(tag) < num_split else 'atleast'
writers_dict[tag].writerow(row)
# Write log
printProgress(count, rec_count, 0.05, start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['RECORDS'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'ParseDb'
printLog(log)
# Close output file handles
for t in handles_dict:
handles_dict[t].close()
return [handles_dict[t].name for t in handles_dict]
def addDbFile(db_file, fields, values, out_args=default_out_args):
"""
Adds field and value pairs to a database file
Arguments:
db_file = the database file name
fields = a list of fields to add
values = a list of values to assign to all rows of each field
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-add',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Define fields and values to append
add_dict = {
k: v
for k, v in zip(fields, values) if k not in db_iter.fieldnames
}
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write updated row
rec.update(add_dict)
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def updateDbFile(db_file, field, values, updates, out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_file = the database file name
field = the field to update
values = a list of values to specifying which rows to update
updates = a list of values to update each value with
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'update'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['VALUES'] = ','.join(values)
log['UPDATES'] = ','.join(updates)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-update',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Updated values if found
for x, y in zip(values, updates):
if rec[field] == x:
rec[field] = y
pass_count += 1
# Write records
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['UPDATED'] = pass_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def tableLog(record_file, fields, out_args=default_out_args):
"""
Converts a pRESTO log to a table of annotations
Arguments:
record_file = the log file name
fields = the list of fields to output
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output table file name
"""
log = OrderedDict()
log['START'] = 'ParseLog'
log['FILE'] = os.path.basename(record_file)
printLog(log)
# Open file handles
log_handle = open(record_file, 'rU')
out_handle = getOutputHandle(record_file,
'table',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over log records
start_time = time()
record = ''
rec_count = pass_count = fail_count = 0
for line in log_handle:
if line.strip() == '' and record:
# Print progress for previous iteration
printProgress(rec_count, None, 1e5, start_time)
# Parse record block
rec_count += 1
record_dict = parseLogRecord(record)
# Write records
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Empty record string
record = ''
else:
# Append to record
record += line
else:
# Write final record
if record:
record_dict = parseLogRecord(record)
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Print counts
printProgress(rec_count, None, 1e5, start_time, end=True)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseLog'
printLog(log)
# Close file handles
log_handle.close()
out_handle.close()
return log_handle.name
def collapseSeq(seq_file, max_missing=default_max_missing, uniq_fields=None,
copy_fields=None, copy_actions=None, max_field=None, min_field=None,
inner=False, keep_missing=False, out_file=None, out_args=default_out_args):
"""
Removes duplicate sequences from a file
Arguments:
seq_file : filename of the sequence file to sample from.
max_missing : number of ambiguous characters to allow in a unique sequence.
uniq_fields : a list of annotations that define a sequence as unique if they differ.
copy_fields : a list of annotations to copy into unique sequence annotations.
copy_actions : the list of collapseAnnotation actions to take on copy_fields.
max_field : a numeric field whose maximum value determines the retained sequence.
min_field : a numeric field whose minimum value determines the retained sequence.
inner : if True exclude consecutive outer ambiguous characters from iterations and matching.
keep_missing : if True retain sequences with more ambiguous characters than max_missing as unique.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: the collapsed output file name.
"""
log = OrderedDict()
log['START'] = 'CollapseSeq'
log['FILE'] = os.path.basename(seq_file)
log['MAX_MISSING'] = max_missing
log['UNIQ_FIELDS'] = ','.join([str(x) for x in uniq_fields]) \
if uniq_fields is not None else None
log['COPY_FIELDS'] = ','.join([str(x) for x in copy_fields]) \
if copy_fields is not None else None
log['COPY_ACTIONS'] = ','.join([str(x) for x in copy_actions]) \
if copy_actions is not None else None
log['MAX_FIELD'] = max_field
log['MIN_FIELD'] = min_field
log['INNER'] = inner
log['KEEP_MISSING'] = keep_missing
printLog(log)
# Read input file
in_type = getFileType(seq_file)
seq_dict = SeqIO.to_dict(readSeqFile(seq_file, index=False))
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Count total sequences
rec_count = len(seq_dict)
# Open unique record output handle
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(seq_file,
'collapse-unique',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
# Find sequences with duplicates
uniq_dict = {}
# Added list typing for compatibility issue with Python 2.7.5 on OS X
# TypeError: object of type 'dictionary-keyiterator' has no len()
search_keys = list(seq_dict.keys())
dup_keys = []
for n in range(0, max_missing + 1):
# Find unique sequences
uniq_dict, search_keys, dup_list = findUniqueSeq(uniq_dict, search_keys, seq_dict, n,
uniq_fields, copy_fields,
max_field, min_field, inner,
out_args['delimiter'])
# Update list of duplicates
dup_keys.extend(dup_list)
# Break if no keys to search remain
if len(search_keys) == 0: break
# Write unique sequences
for val in uniq_dict.values():
# Define output sequence
out_seq = val.seq
out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter'])
out_app = OrderedDict()
if copy_fields is not None and copy_actions is not None:
for f, a in zip(copy_fields, copy_actions):
x = collapseAnnotation(val.annotations, a, f, delimiter=out_args['delimiter'])
out_app[f] = x[f]
out_ann.pop(f, None)
out_app['DUPCOUNT'] = val.count
out_ann = mergeAnnotation(out_ann, out_app, delimiter=out_args['delimiter'])
out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter'])
out_seq.description = ''
# Write unique sequence
SeqIO.write(out_seq, pass_handle, out_args['out_type'])
# Update log
log = OrderedDict()
log['HEADER'] = out_seq.id
log['DUPCOUNT'] = val.count
for i, k in enumerate(val.keys, start=1):
log['ID%i' % i] = k
for i, k in enumerate(val.keys, start=1):
log['SEQ%i' % i] = str(seq_dict[k].seq)
printLog(log, handle=log_handle)
# Write sequence with high missing character counts
if keep_missing:
for k in search_keys:
out_seq = seq_dict[k]
out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter'])
out_ann = mergeAnnotation(out_ann, {'DUPCOUNT':1}, delimiter=out_args['delimiter'])
out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter'])
out_seq.description = ''
SeqIO.write(out_seq, pass_handle, out_args['out_type'])
# Write sequence with high missing character counts
if out_args['failed'] and not keep_missing:
with getOutputHandle(seq_file, 'collapse-undetermined', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type']) \
as missing_handle:
for k in search_keys:
SeqIO.write(seq_dict[k], missing_handle, out_args['out_type'])
if out_args['failed']:
# Write duplicate sequences
with getOutputHandle(seq_file, 'collapse-duplicate', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type']) \
as dup_handle:
for k in dup_keys:
SeqIO.write(seq_dict[k], dup_handle, out_args['out_type'])
# Print log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['SEQUENCES'] = rec_count
log['UNIQUE'] = len(uniq_dict)
log['DUPLICATE'] = len(dup_keys)
log['UNDETERMINED'] = len(search_keys)
log['END'] = 'CollapseSeq'
printLog(log)
# Close file handles
pass_file = pass_handle.name
if pass_handle is not None: pass_handle.close()
if log_handle is not None: log_handle.close()
return pass_file
def selectDbFile(db_file,
fields,
values,
logic='any',
regex=False,
out_args=default_out_args):
"""
Selects records from a database file
Arguments:
db_file = the database file name
fields = a list of fields to check for selection criteria
values = a list of values defining selection targets
logic = one of 'any' or 'all' defining whether one or all fields must have a match.
regex = if False do exact full string matches; if True allow partial regex matches.
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define string match function
if regex:
def _match_func(x, patterns):
return any([re.search(p, x) for p in patterns])
else:
def _match_func(x, patterns):
return x in patterns
# Define logic function
if logic == 'any':
_logic_func = any
elif logic == 'all':
_logic_func = all
# Print console log
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'select'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
log['REGEX'] = regex
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-select',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Check for selection values in all fields
select = _logic_func(
[_match_func(rec.get(f, False), values) for f in fields])
# Write sequences
if select:
pass_count += 1
pass_writer.writerow(rec)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['SELECTED'] = pass_count
log['DISCARDED'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def tableLog(record_file, fields, out_file=None, out_args=default_out_args):
"""
Converts a pRESTO log to a table of annotations
Arguments:
record_file : the log file name.
fields : the list of fields to output.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: the output table file name
"""
log = OrderedDict()
log['START'] = 'ParseLog'
log['FILE'] = os.path.basename(record_file)
printLog(log)
# Open file handles
log_handle = open(record_file)
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(record_file,
'table',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle,
extrasaction='ignore',
restval='',
delimiter='\t',
fieldnames=fields)
out_writer.writeheader()
# Iterate over log records
start_time = time()
record = ''
rec_count = pass_count = fail_count = 0
for line in log_handle:
if line.strip() == '' and record:
# Print progress for previous iteration
printCount(rec_count, 1e5, start_time=start_time)
# Parse record block
rec_count += 1
record_dict = parseLog(record)
# Write records
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Empty record string
record = ''
else:
# Append to record
record += line
else:
# Write final record
if record:
record_dict = parseLog(record)
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Print counts
printCount(rec_count, 1e5, start_time, end=True)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseLog'
printLog(log)
# Close file handles
log_handle.close()
out_handle.close()
return log_handle.name
def sortDbFile(db_file,
field,
numeric=False,
descend=False,
out_args=default_out_args):
"""
Sorts records by values in an annotation field
Arguments:
db_file = the database filename
field = the field name to sort by
numeric = if True sort field numerically;
if False sort field alphabetically
descend = if True sort in descending order;
if False sort in ascending order
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-sort',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Store all records in a dictionary
start_time = time()
printMessage("Indexing: Running", start_time=start_time)
db_dict = {i: r for i, r in enumerate(db_iter)}
result_count = len(db_dict)
# Sort db_dict by field values
tag_dict = {k: v[field] for k, v in db_dict.items()}
if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend)
printMessage("Indexing: Done", start_time=start_time, end=True)
# Iterate over records
start_time = time()
rec_count = 0
for key in sorted_keys:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write records
pass_writer.writerow(db_dict[key])
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def groupSeqFile(seq_file, field, threshold=None, out_args=default_out_args):
"""
Divides a sequence file into segments by description tags
Arguments:
seq_file : filename of the sequence file to split
field : The annotation field to split seq_file by
threshold : The numerical threshold for group sequences by;
if None treat field as textual
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'group'
log['FILE'] = os.path.basename(seq_file)
log['FIELD'] = field
log['THRESHOLD'] = threshold
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Determine total numbers of records
rec_count = countSeqFile(seq_file)
# Process sequences
start_time = time()
seq_count = 0
if threshold is None:
# Sort records into files based on textual field
# Create set of unique field tags
temp_iter = readSeqFile(seq_file)
tag_list = getAnnotationValues(temp_iter,
field,
unique=True,
delimiter=out_args['delimiter'])
if sys.platform != 'win32':
import resource
# Increase open file handle limit if needed
file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
file_count = len(tag_list) + 256
if file_limit < file_count and file_count <= 8192:
#print file_limit, file_count
resource.setrlimit(resource.RLIMIT_NOFILE,
(file_count, file_count))
elif file_count > 8192:
e = '''OS file limit would need to be set to %i.
If you are sure you want to do this, then increase the
file limit in the OS (via ulimit) and rerun this tool.
''' % file_count
printError(dedent(e))
# Create output handles
# out_label = '%s=%s' % (field, tag)
handles_dict = {
tag: getOutputHandle(seq_file,
'%s-%s' % (field, tag),
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
for tag in tag_list
}
# Iterate over sequences
for seq in seq_iter:
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write sequences
tag = parseAnnotation(seq.description,
delimiter=out_args['delimiter'])[field]
SeqIO.write(seq, handles_dict[tag], out_args['out_type'])
else:
# Sort records into files based on numeric threshold
threshold = float(threshold)
# Create output handles
handles_dict = {
'under':
getOutputHandle(seq_file,
'under-%.1g' % threshold,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type']),
'atleast':
getOutputHandle(seq_file,
'atleast-%.1g' % threshold,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
}
# Iterate over sequences
for seq in seq_iter:
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write sequences
tag = parseAnnotation(seq.description,
delimiter=out_args['delimiter'])[field]
tag = 'under' if float(tag) < threshold else 'atleast'
SeqIO.write(seq, handles_dict[tag], out_args['out_type'])
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['SEQUENCES'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'SplitSeq'
printLog(log)
# Close output file handles
for k in handles_dict:
handles_dict[k].close()
return [handles_dict[k].name for k in handles_dict]
def pairSeq(seq_file_1, seq_file_2, fields_1=None, fields_2=None,
coord_type=default_coord_type,
out_args=default_out_args):
"""
Generates consensus sequences
Arguments:
seq_file_1 = the file containing the grouped sequences and annotations
seq_file_2 = the file to assign annotations to from seq_file_1
fields_1 = list of annotations in seq_file_1 records to copy to seq_file_2 records;
if None do not copy any annotations
fields_2 = list of annotations in seq_file_2 records to copy to seq_file_1 records;
if None do not copy any annotations
coord_type = the sequence header format
out_args = common output argument dictionary from parseCommonArgs
Returns:
a list of tuples holding successfully paired filenames for (seq_file_1, seq_file_2)
"""
# Define private functions
def _key_func(x):
return getCoordKey(x, coord_type=coord_type, delimiter=out_args['delimiter'])
log = OrderedDict()
log['START'] = 'PairSeq'
log['FILE1'] = os.path.basename(seq_file_1)
log['FILE2'] = os.path.basename(seq_file_2)
log['FIELDS_1'] = ','.join(fields_1) if fields_1 is not None else None
log['FIELDS_2'] = ','.join(fields_2) if fields_2 is not None else None
log['COORD_TYPE'] = coord_type
printLog(log)
# Define output type
if out_args['out_type'] is None:
out_type_1 = getFileType(seq_file_1)
out_type_2 = getFileType(seq_file_2)
else:
out_type_1 = out_type_2 = out_args['out_type']
# Define output name
if out_args['out_name'] is None:
out_name_1 = out_name_2 = None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
# Open and count files
start_time = time()
printMessage("Indexing files", start_time=start_time)
# Index file 1
seq_count_1 = countSeqFile(seq_file_1)
seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func)
# Define file 2 iterator
seq_count_2 = countSeqFile(seq_file_2)
seq_iter_2 = readSeqFile(seq_file_2, index=False)
printMessage("Done", start_time=start_time, end=True)
# Open output file handles
pass_handle_1 = getOutputHandle(seq_file_1, 'pair-pass', out_args['out_dir'],
out_name=out_name_1, out_type=out_type_1)
pass_handle_2 = getOutputHandle(seq_file_2, 'pair-pass', out_args['out_dir'],
out_name=out_name_2, out_type=out_type_2)
if out_args['failed']:
fail_handle_1 = getOutputHandle(seq_file_1, 'pair-fail', out_dir=out_args['out_dir'],
out_name=out_name_1, out_type=out_type_1)
fail_handle_2 = getOutputHandle(seq_file_2, 'pair-fail', out_dir=out_args['out_dir'],
out_name=out_name_2, out_type=out_type_2)
pass_keys = list()
# Iterate over pairs and write to output files
start_time = time()
rec_count = pair_count = 0
for seq_2 in seq_iter_2:
# Print progress for previous iteration
printProgress(rec_count, seq_count_2, 0.05, start_time)
rec_count += 1
# Check for file 2 mate pair in file 1
coord_2 = getCoordKey(seq_2.id, coord_type=coord_type,
delimiter=out_args['delimiter'])
seq_1 = seq_dict_1.get(coord_2, None)
if seq_1 is not None:
# Record paired keys
pair_count += 1
if fields_1 is not None or fields_2 is not None:
ann_1 = parseAnnotation(seq_1.description,
delimiter=out_args['delimiter'])
ann_2 = parseAnnotation(seq_2.description,
delimiter=out_args['delimiter'])
# Prepend annotations from seq_1 to seq_2
if fields_1 is not None:
copy_ann = OrderedDict([(k, v) for k, v in ann_1.items() \
if k in fields_1])
merge_ann = mergeAnnotation(ann_2, copy_ann, prepend=True,
delimiter=out_args['delimiter'])
seq_2.id = flattenAnnotation(merge_ann,
delimiter=out_args['delimiter'])
seq_2.description = ''
# Append annotations from seq_2 to seq_1
if fields_2 is not None:
copy_ann = OrderedDict([(k, v) for k, v in ann_2.items() \
if k in fields_2])
merge_ann = mergeAnnotation(ann_1, copy_ann, prepend=False,
delimiter=out_args['delimiter'])
seq_1.id = flattenAnnotation(merge_ann,
delimiter=out_args['delimiter'])
seq_1.description = ''
# Write paired records
SeqIO.write(seq_1, pass_handle_1, out_type_1)
SeqIO.write(seq_2, pass_handle_2, out_type_2)
# Write unpaired file 2 records and updated paired key list for finding unpaired file 1 records
if out_args['failed']:
if seq_1 is not None: pass_keys.append(coord_2)
else: SeqIO.write(seq_2, fail_handle_2, out_type_2)
# Print final progress
printProgress(rec_count, seq_count_2, 0.05, start_time)
# Find and write unpaired file 1 records
if out_args['failed']:
start_time = time()
printMessage("Finding unpaired", start_time=start_time)
# Find file 1 unpaired keys
pass_keys = set(pass_keys)
unpaired = set(seq_dict_1).difference(pass_keys)
# Write unpaired file 1 records
for k in unpaired: SeqIO.write(seq_dict_1[k], fail_handle_1, out_type_1)
printMessage("Done", start_time=start_time, end=True)
# Print log
log = OrderedDict()
log['OUTPUT1'] = os.path.basename(pass_handle_1.name)
log['OUTPUT2'] = os.path.basename(pass_handle_2.name)
log['SEQUENCES1'] = seq_count_1
log['SEQUENCES2'] = seq_count_2
log['PASS'] = pair_count
log['END'] = 'PairSeq'
printLog(log)
# Close file handles
pass_handle_1.close()
pass_handle_2.close()
return [(pass_handle_1.name, pass_handle_2.name)]
def sampleSeqFile(seq_file,
max_count,
field=None,
values=None,
out_args=default_out_args):
"""
Samples from a sequence file
Arguments:
seq_file : filename of the sequence file to sample from
max_count : a list of the maximum number of sequences to sample
field : the annotation field to check for required values
values : a list of annotation values that a sample must contain one of
out_args : common output argument dictionary from parseCommonArgs
Returns:
str: output file name
"""
# Function to sample from a list of sequence indices
def _sample_list(n, index_list):
max_n = len(index_list)
r = random.sample(range(max_n), n) if n < max_n else range(max_n)
return [index_list[x] for x in r]
# Function to sample from a dictionary of grouped sequence indices
def _sample_dict(n, index_dict):
sample_list = []
for v in index_dict.values():
max_n = len(v)
r = random.sample(range(max_n), n) if n < max_n else range(max_n)
sample_list.extend([v[x] for x in r])
return sample_list
# Print console log
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'sample'
log['FILE'] = os.path.basename(seq_file)
log['MAX_COUNTS'] = ','.join([str(x) for x in max_count])
log['FIELD'] = field
log['VALUES'] = ','.join(values) if values else None
printLog(log)
# Read input files and open output files
start_time = time()
printMessage('Reading files', start_time=start_time, width=25)
in_type = getFileType(seq_file)
seq_dict = readSeqFile(seq_file, index=True)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Generate subset of records
if field is not None and values is not None:
_sample = _sample_list
printMessage('Subsetting by annotation',
start_time=start_time,
width=25)
seq_index = subsetSeqIndex(seq_dict,
field,
values,
delimiter=out_args['delimiter'])
elif field is not None and values is None:
_sample = _sample_dict
printMessage('Indexing by annotation', start_time=start_time, width=25)
seq_index = indexSeqSets(seq_dict,
field,
delimiter=out_args['delimiter'])
else:
_sample = _sample_list
seq_index = [k for k in seq_dict]
printMessage('Done', start_time=start_time, end=True, width=25)
# Generate sample set for each value in max_count
out_files = []
for i, n in enumerate(max_count):
start_time = time()
printMessage('Sampling n=%i' % n, start_time=start_time, width=25)
# Sample from records
sample_keys = _sample(n, seq_index)
sample_count = len(sample_keys)
# Write sampled sequences to files
with getOutputHandle(seq_file,
'sample%i-n%i' % (i + 1, sample_count),
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type']) as out_handle:
for k in sample_keys:
SeqIO.write(seq_dict[k], out_handle, out_args['out_type'])
out_files.append(out_handle.name)
printMessage('Done', start_time=start_time, end=True, width=25)
# Print log for iteration
log = OrderedDict()
log['MAX_COUNT'] = n
log['SAMPLED'] = sample_count
log['OUTPUT'] = os.path.basename(out_files[i])
printLog(log)
# Print log
log = OrderedDict()
log['END'] = 'SplitSeq'
printLog(log)
return out_files
def collectQueue(alive, result_queue, collect_queue, db_file, out_args, cluster_func=None, cluster_args={}):
"""
Assembles results from a queue of individual sequence results and manages log/file I/O
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing continues
if False exit process
result_queue = a multiprocessing.Queue holding processQueue results
collect_queue = a multiprocessing.Queue to store collector return values
db_file = the input database file name
out_args = common output argument dictionary from parseCommonArgs
cluster_func = the function to call for carrying out clustering on distance matrix
cluster_args = a dictionary of arguments to pass to cluster_func
Returns:
None
(adds 'log' and 'out_files' to collect_dict)
"""
# Open output files
try:
# Count records and define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
result_count = countDbFile(db_file)
# Defined successful output handle
pass_handle = getOutputHandle(db_file,
out_label='clone-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE')
# Defined failed alignment output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
out_label='clone-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
fail_writer = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
#sys.stderr.write('Exception in collector file opening step\n')
alive.value = False
raise
# Get results from queue and write to files
try:
#print 'START COLLECT', alive.value
# Iterator over results queue until sentinel object reached
start_time = time()
rec_count = clone_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
#print "COLLECT", alive.value, result['id']
# Print progress for previous iteration and update record count
if rec_count == 0:
print('PROGRESS> Assigning clones')
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += len(result.data)
# Write passed and failed records
if result:
for clone in result.results.values():
clone_count += 1
for i, rec in enumerate(clone):
rec.annotations['CLONE'] = clone_count
pass_writer.writerow(rec.toDict())
pass_count += 1
result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction)
else:
for i, rec in enumerate(result.data):
if fail_writer is not None: fail_writer.writerow(rec.toDict())
fail_count += 1
result.log['CLONE0-%i' % (i + 1)] = str(rec.junction)
# Write log
printLog(result.log, handle=log_handle)
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(rec_count, result_count, 0.05, start_time)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
# Update return list
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
except:
#sys.stderr.write('Exception in collector result processing step\n')
alive.value = False
raise
return None
def downsizeSeqFile(seq_file, max_count, out_args=default_out_args):
"""
Splits a FASTA/FASTQ file into segments with a limited number of records
Arguments:
seq_file : filename of the FASTA file to split
max_count : number of records in each output file
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'count'
log['FILE'] = os.path.basename(seq_file)
log['MAX_COUNT'] = max_count
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Determine total numbers of records
rec_count = countSeqFile(seq_file)
# Loop through iterator writing each record and opening new output handle as needed
start_time = time()
seq_count, part_num = 0, 1
out_handle = getOutputHandle(seq_file,
'part%06i' % part_num,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files = [out_handle.name]
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
# Update count
seq_count += 1
# Write records
SeqIO.write(seq, out_handle, out_args['out_type'])
# Break if total records reached to avoid extra empty file
if seq_count == rec_count:
break
# Open new file if needed
if seq_count % max_count == 0:
out_handle.close()
part_num += 1
out_handle = getOutputHandle(seq_file,
'part%06i' % part_num,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files.append(out_handle.name)
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, f in enumerate(out_files):
log['OUTPUT%i' % (i + 1)] = os.path.basename(f)
log['SEQUENCES'] = rec_count
log['PARTS'] = len(out_files)
log['END'] = 'SplitSeq'
printLog(log)
# Close file handles
out_handle.close()
return out_files
def convertHeaders(seq_file, convert_func, convert_args={}, out_args=default_out_args):
"""
Converts sequence headers to the pRESTO format
Arguments:
seq_file = the sequence file name
convert_func = the function used to convert sequence headers
convert_args = a dictionary of arguments to pass to convert_func
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output sequence file name
"""
# Define subcommand label dictionary
cmd_dict = {convertGenericHeader:'generic',
convert454Header:'454',
convertGenbankHeader:'genbank',
convertIlluminaHeader:'illumina',
convertIMGTHeader:'imgt',
convertSRAHeader:'sra'}
log = OrderedDict()
log['START'] = 'ConvertHeaders'
log['COMMAND'] = cmd_dict[convert_func]
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open input file
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Count records
result_count = countSeqFile(seq_file)
# Open output file handles
pass_handle = getOutputHandle(seq_file,
'convert-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
if out_args['failed']:
fail_handle = getOutputHandle(seq_file,
'convert-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
else:
fail_handle = None
# Set additional conversion arguments
if convert_func in [convertGenericHeader, convertGenbankHeader]:
convert_args.update({'delimiter':out_args['delimiter']})
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration and update count
printProgress(seq_count, result_count, 0.05, start_time)
seq_count += 1
# Convert header
header = convert_func(seq.description, **convert_args)
if header is not None:
# Write successfully converted sequences
pass_count += 1
seq.id = seq.name = flattenAnnotation(header, out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, pass_handle, out_args['out_type'])
else:
fail_count += 1
if fail_handle is not None:
# Write successfully unconverted sequences
SeqIO.write(seq, fail_handle, out_args['out_type'])
# Print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ConvertHeaders'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
return pass_handle.name
def samplePairSeqFile(seq_file_1,
seq_file_2,
max_count,
field=None,
values=None,
coord_type=default_coord,
out_args=default_out_args):
"""
Samples from paired-end sequence files
Arguments:
seq_file_1 : filename of the first paired-end sequence file
seq_file_2 : filename of the second paired-end sequence file
max_count : a list of the maximum number of sequences to sample
field : the annotation field to check for required values
values : a list of annotation values that a sample must contain one of
coord_type : the sequence header format
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: seq_file_1 and seq_file_2 output file names
"""
# Sequence index key function
def _key_func(x):
return getCoordKey(x,
coord_type=coord_type,
delimiter=out_args['delimiter'])
# Function to sample from two lists of sequence indices
def _sample_list(n, index_1, index_2):
key_set = set(index_1).intersection(index_2)
max_n = len(key_set)
return random.sample(key_set, min(n, max_n))
# Function to sample from two dictionaries of grouped sequence indices
def _sample_dict(n, index_1, index_2):
group_set = set(index_1.keys()).intersection(index_2.keys())
sample_list = []
for k in group_set:
key_set = set(index_1[k]).intersection(index_2[k])
max_n = len(key_set)
sample_list.extend(random.sample(key_set, min(n, max_n)))
return sample_list
# Print console log
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'samplepair'
log['FILE1'] = os.path.basename(seq_file_1)
log['FILE2'] = os.path.basename(seq_file_2)
log['MAX_COUNTS'] = ','.join([str(x) for x in max_count])
log['FIELD'] = field
log['VALUES'] = ','.join(values) if values else None
printLog(log)
# Define output type
in_type_1 = getFileType(seq_file_1)
in_type_2 = getFileType(seq_file_2)
if out_args['out_type'] is None:
out_type_1 = in_type_1
out_type_2 = in_type_2
else:
out_type_1 = out_type_2 = out_args['out_type']
# Define output name
if out_args['out_name'] is None:
out_name_1 = out_name_2 = None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
# Index input files
start_time = time()
printMessage('Reading files', start_time=start_time, width=25)
seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func)
seq_dict_2 = readSeqFile(seq_file_2, index=True, key_func=_key_func)
# Subset keys to those meeting field/value criteria
if field is not None and values is not None:
_sample = _sample_list
printMessage('Subsetting by annotation',
start_time=start_time,
width=25)
seq_index_1 = subsetSeqIndex(seq_dict_1,
field,
values,
delimiter=out_args['delimiter'])
seq_index_2 = subsetSeqIndex(seq_dict_2,
field,
values,
delimiter=out_args['delimiter'])
elif field is not None and values is None:
_sample = _sample_dict
printMessage('Indexing by annotation', start_time=start_time, width=25)
seq_index_1 = indexSeqSets(seq_dict_1,
field,
delimiter=out_args['delimiter'])
seq_index_2 = indexSeqSets(seq_dict_2,
field,
delimiter=out_args['delimiter'])
else:
_sample = _sample_list
seq_index_1 = list(seq_dict_1.keys())
seq_index_2 = list(seq_dict_2.keys())
printMessage('Done', start_time=start_time, end=True, width=25)
# Generate sample set for each value in max_count
out_files = []
for i, n in enumerate(max_count):
start_time = time()
printMessage('Sampling n=%i' % n, start_time=start_time, width=25)
# Sample
sample_keys = _sample(n, seq_index_1, seq_index_2)
sample_count = len(sample_keys)
# Open file handles
out_handle_1 = getOutputHandle(seq_file_1,
'sample%i-n%i' % (i + 1, sample_count),
out_dir=out_args['out_dir'],
out_name=out_name_1,
out_type=out_type_1)
out_handle_2 = getOutputHandle(seq_file_2,
'sample%i-n%i' % (i + 1, sample_count),
out_dir=out_args['out_dir'],
out_name=out_name_2,
out_type=out_type_2)
out_files.append((out_handle_1.name, out_handle_2.name))
for k in sample_keys:
SeqIO.write(seq_dict_1[k], out_handle_1, out_type_1)
SeqIO.write(seq_dict_2[k], out_handle_2, out_type_2)
printMessage('Done', start_time=start_time, end=True, width=25)
# Print log for iteration
log = OrderedDict()
log['MAX_COUNT'] = n
log['SAMPLED'] = sample_count
log['OUTPUT1'] = os.path.basename(out_files[i][0])
log['OUTPUT2'] = os.path.basename(out_files[i][1])
printLog(log)
# Close file handles
out_handle_1.close()
out_handle_2.close()
# Print log
log = OrderedDict()
log['END'] = 'SplitSeq'
printLog(log)
return out_files
def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field,
meta_fields=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sequences
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
out_type = 'fasta'
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_type)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Get SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter'])
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, out_type)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def sortSeqFile(seq_file,
field,
numeric=False,
max_count=None,
out_args=default_out_args):
"""
Sorts a sequence file by annotation fields
Arguments:
seq_file : filename of the sequence file to split
field : position of field in sequence description to split by
numeric : if True sort field numerically;
if False sort field alphabetically
max_count : maximum number of records in each output file
if None do not create multiple files
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(seq_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
log['MAX_COUNT'] = max_count
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_dict = readSeqFile(seq_file, index=True)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Get annotations and sort seq_dict by annotation values
tag_dict = {
k: parseAnnotation(seq_dict[k].description,
delimiter=out_args['delimiter'])[field]
for k in seq_dict
}
if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get)
# Determine total numbers of records
rec_count = len(seq_dict)
if max_count >= rec_count: max_count = None
# Open initial output file handles
file_count = 1
if max_count is None: out_label = 'sorted'
else: out_label = 'sorted-part%06i' % file_count
out_handle = getOutputHandle(seq_file,
out_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files = [out_handle.name]
# Loop through sorted sequence dictionary keys
start_time = time()
last_tag = None
saved_keys = []
seq_count = chunk_count = 0
for key in sorted_keys:
# Print progress for previous iteration and update count
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write saved group of sequences when tag changes
if last_tag is not None and tag_dict[key] != last_tag:
# Open new output file if needed
if max_count is not None and chunk_count + len(
saved_keys) > max_count:
# Update partition counts
file_count += 1
chunk_count = 0
# Open new file handle
out_handle.close()
out_handle = getOutputHandle(seq_file,
'sorted-part%06i' % file_count,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Append output file name to out_files
out_files.append(out_handle.name)
# Write saved sequences
for k in saved_keys:
chunk_count += 1
SeqIO.write(seq_dict[k], out_handle, out_args['out_type'])
# Reset saved keys to current key only
saved_keys = [key]
else:
# Update list of saved keys if tag is unchanged
saved_keys.append(key)
# Check if total records reached, write all saved keys, and exit loop
if seq_count == rec_count:
for k in saved_keys:
chunk_count += 1
SeqIO.write(seq_dict[k], out_handle, out_args['out_type'])
out_handle.close()
break
# Update tag tracker
last_tag = tag_dict[key]
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, f in enumerate(out_files):
log['OUTPUT%i' % (i + 1)] = os.path.basename(f)
log['SEQUENCES'] = seq_count
log['PARTS'] = len(out_files)
log['END'] = 'SplitSeq'
printLog(log)
# Close file handles
out_handle.close()
return out_files
def selectSeqFile(seq_file,
field,
value_list=None,
value_file=None,
negate=False,
out_file=None,
out_args=default_out_args):
"""
Select from a sequence file
Arguments:
seq_file : filename of the sequence file to sample from.
field : the annotation field to check for required values.
value_list : a list of annotation values that a sample must contain one of.
value_file : a tab delimited file containing values to select.
negate : if True select entires that do not contain the specific values.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: output file name.
"""
# Reads value_file
def _read_file(value_file, field):
field_list = []
try:
with open(value_file, 'rt') as handle:
reader_dict = csv.DictReader(handle, dialect='excel-tab')
for row in reader_dict:
field_list.append(row[field])
except IOError:
printError('File %s cannot be read.' % value_file)
except:
printError('File %s is invalid.' % value_file)
return field_list
# Print console log
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'select'
log['FILE'] = os.path.basename(seq_file)
log['FIELD'] = field
if value_list is not None:
log['VALUE_LIST'] = ','.join([str(x) for x in value_list])
if value_file is not None:
log['VALUE_FILE'] = os.path.basename(value_file)
log['NOT'] = negate
printLog(log)
# Read value_file
if value_list is not None and value_file is not None:
printError('Specify only one of value_list and value_file.')
elif value_file is not None:
value_list = _read_file(value_file, field)
# Read sequence file
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Output output handle
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(seq_file,
'selected',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Generate subset of records
start_time = time()
pass_count, fail_count, rec_count = 0, 0, 0
value_set = set(value_list)
for rec in seq_iter:
printCount(rec_count, 1e5, start_time=start_time)
rec_count += 1
# Parse annotations into a list of values
ann = parseAnnotation(rec.description,
delimiter=out_args['delimiter'])[field]
ann = ann.split(out_args['delimiter'][2])
# Write
if xor(negate, not value_set.isdisjoint(ann)):
# Write
SeqIO.write(rec, out_handle, out_args['out_type'])
pass_count += 1
else:
fail_count += 1
printCount(rec_count, 1e5, start_time=start_time, end=True)
# Print log
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'SplitSeq'
printLog(log)
return out_handle.name
def modifyHeaders(seq_file, modify_func, modify_args, out_args=default_out_args):
"""
Modifies sequence headers
Arguments:
seq_file = the sequence file name
modify_func = the function defining the modification operation
modify_args = a dictionary of arguments to pass to modify_func
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define subcommand label dictionary
cmd_dict = {addHeader: 'add',
copyHeader: 'copy',
collapseHeader: 'collapse',
deleteHeader: 'delete',
expandHeader: 'expand',
renameHeader: 'rename'}
# Print parameter info
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__)
log['FILE'] = os.path.basename(seq_file)
for k in sorted(modify_args):
v = modify_args[k]
log[k.upper()] = ','.join(v) if isinstance(v, list) else v
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
# Count records
result_count = countSeqFile(seq_file)
# Iterate over sequences
start_time = time()
seq_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time)
#Update counts
seq_count += 1
# Modify header
header = parseAnnotation(seq.description, delimiter=out_args['delimiter'])
header = modify_func(header, delimiter=out_args['delimiter'], **modify_args)
# Write new sequence
seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, out_handle, out_args['out_type'])
# print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args):
"""
Renames fields in a database file
Arguments:
db_file = the database file name
fields = a list of fields to rename
values = a list of new names for fields
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'rename'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['NAMES'] = ','.join(names)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
# Get header and rename fields
header = (readDbFile(db_file, ig=False)).fieldnames
for f, n in zip(fields, names):
i = header.index(f)
header[i] = n
# Open writer and write new header
# TODO: should modify getDbWriter to take a list of fields
pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab')
pass_writer.writeheader()
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore
# Rename fields
for f, n in zip(fields, names):
rec[n] = rec.pop(f)
# Write
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def tableHeaders(seq_file, fields, out_args=default_out_args):
"""
Builds a table of sequence header annotations
Arguments:
seq_file = the sequence file name
fields = the list of fields to output
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output table file name
"""
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open file handles
seq_iter = readSeqFile(seq_file)
out_handle = getOutputHandle(seq_file, out_label='headers', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
# Count records
result_count = countSeqFile(seq_file)
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time)
# Get annotations
seq_count += 1
ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter'])
# Write records
if ann:
pass_count += 1
out_writer.writerow(ann)
else:
fail_count += 1
# Print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args):
"""
Divides a tab-delimited database file into segments by description tags
Arguments:
db_file = filename of the tab-delimited database file to split
field = the field name by which to split db_file
num_split = the numerical threshold by which to group sequences;
if None treat field as textual
out_args = common output argument dictionary from parseCommonArgs
Returns:
a list of output file names
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'split'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUM_SPLIT'] = num_split
printLog(log)
# Open IgRecord reader iter object
reader = readDbFile(db_file, ig=False)
# Determine total numbers of records
rec_count = countDbFile(db_file)
start_time = time()
count = 0
# Sort records into files based on textual field
if num_split is None:
# Create set of unique field tags
tmp_iter = readDbFile(db_file, ig=False)
tag_list = list(set([row[field] for row in tmp_iter]))
# Forbidden characters in filename and replacements
noGood = {'\/':'f','\\':'b','?':'q','\%':'p','*':'s',':':'c',
'\|':'pi','\"':'dq','\'':'sq','<':'gt','>':'lt',' ':'_'}
# Replace forbidden characters in tag_list
tag_dict = {}
for tag in tag_list:
for c,r in noGood.items():
tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \
if c in tag else tag_dict.get(tag, tag))
# Create output handles
handles_dict = {tag:getOutputHandle(db_file,
'%s-%s' % (field, label),
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir'])
for tag, label in tag_dict.items()}
# Create Db writer instances
writers_dict = {tag:getDbWriter(handles_dict[tag], db_file)
for tag in tag_dict}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
# Write row to appropriate file
tag = row[field]
writers_dict[tag].writerow(row)
# Sort records into files based on numeric num_split
else:
num_split = float(num_split)
# Create output handles
handles_dict = {'under':getOutputHandle(db_file,
'under-%.1f' % num_split,
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir']),
'atleast':getOutputHandle(db_file,
'atleast-%.1f' % num_split,
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir'])}
# Create Db writer instances
writers_dict = {'under':getDbWriter(handles_dict['under'], db_file),
'atleast':getDbWriter(handles_dict['atleast'], db_file)}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
tag = row[field]
tag = 'under' if float(tag) < num_split else 'atleast'
writers_dict[tag].writerow(row)
# Write log
printProgress(count, rec_count, 0.05, start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['RECORDS'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'ParseDb'
printLog(log)
# Close output file handles
for t in handles_dict: handles_dict[t].close()
return [handles_dict[t].name for t in handles_dict]
def assembleEachGermline(db_file,
repo,
germ_types,
v_field,
seq_field,
out_args=default_out_args):
"""
Write germline sequences to tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types,
str) else ','.join(germ_types)
log['CLONED'] = 'False'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
references = readRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS']
# Create output file handle and Db writer
pass_handle = getOutputHandle(db_file,
'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
fail_writer = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
pass_count = fail_count = 0
# Iterate over rows
for i, row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
result_log, germlines = joinGermline(row,
references,
seq_field=seq_field,
v_field=v_field,
germ_types=germ_types)
# Add germline field(s) to dictionary
if 'full' in germ_types:
row['GERMLINE_' + seq_type] = germlines['full']
if 'dmask' in germ_types:
row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask']
if 'vonly' in germ_types:
row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly']
if 'regions' in germ_types:
row['GERMLINE_REGIONS'] = germlines['regions']
# Write row to pass or fail file
if 'ERROR' in result_log:
fail_count += 1
if fail_writer is not None: fail_writer.writerow(row)
else:
result_log['SEQUENCE'] = row[seq_field]
result_log['GERMLINE'] = germlines['full']
result_log['REGIONS'] = germlines['regions']
pass_count += 1
pass_writer.writerow(row)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i + 1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def updateDbFile(db_file, field, values, updates, out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_file = the database file name
field = the field to update
values = a list of values to specifying which rows to update
updates = a list of values to update each value with
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'update'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['VALUES'] = ','.join(values)
log['UPDATES'] = ','.join(updates)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Updated values if found
for x, y in zip(values, updates):
if rec[field] == x:
rec[field] = y
pass_count += 1
# Write records
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['UPDATED'] = pass_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args,
cluster_func, cluster_args):
"""
Assembles results from a queue of individual sequence results and manages log/file I/O
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing continues
if False exit process
result_queue = a multiprocessing.Queue holding processQueue results
collect_queue = a multiprocessing.Queue to store collector return values
db_file = the input database file name
out_args = common output argument dictionary from parseCommonArgs
cluster_func = the function to call for carrying out clustering on distance matrix
cluster_args = a dictionary of arguments to pass to cluster_func
Returns:
None
(adds 'log' and 'out_files' to collect_dict)
"""
# Open output files
try:
# Iterate over Ig records to count and order by junction length
result_count = 0
records = {}
# print 'Reading file...'
db_iter = readDbFile(db_file)
for rec in db_iter:
records[rec.id] = rec
result_count += 1
records = OrderedDict(
sorted(list(records.items()), key=lambda i: i[1].junction_length))
# Define empty matrix to store assembled results
dist_mat = np.zeros((result_count, result_count))
# Count records and define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined successful output handle
pass_handle = getOutputHandle(db_file,
out_label='clone-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE')
# Defined failed cloning output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
out_label='clone-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
fail_writer = None
# Open log file
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
row_count = rec_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
if row_count == 0:
print('PROGRESS> Assigning clones')
printProgress(row_count, result_count, 0.05, start_time)
# Update counts for iteration
row_count += 1
rec_count += len(result)
# Add result row to distance matrix
if result:
dist_mat[list(range(result_count - len(result), result_count)),
result_count - len(result)] = result.results
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Calculate linkage and carry out clustering
# print dist_mat
clusters = cluster_func(dist_mat, **
cluster_args) if dist_mat is not None else None
clones = {}
# print clusters
for i, c in enumerate(clusters):
clones.setdefault(c, []).append(records[list(records.keys())[i]])
# Write passed and failed records
clone_count = pass_count = fail_count = 0
if clones:
for clone in clones.values():
clone_count += 1
for i, rec in enumerate(clone):
rec.annotations['CLONE'] = clone_count
pass_writer.writerow(rec.toDict())
pass_count += 1
#result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction)
else:
for i, rec in enumerate(result.data):
fail_writer.writerow(rec.toDict())
fail_count += 1
#result.log['CLONE0-%i' % (i + 1)] = str(rec.junction)
# Print final progress
printProgress(row_count, result_count, 0.05, start_time)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
# Update return list
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log': log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
