def addDbFile(db_file, fields, values, out_args=default_out_args):
"""
Adds field and value pairs to a database file
Arguments:
db_file = the database file name
fields = a list of fields to add
values = a list of values to assign to all rows of each field
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Define fields and values to append
add_dict = {k:v for k,v in zip(fields, values) if k not in db_iter.fieldnames}
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write updated row
rec.update(add_dict)
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args):
"""
Adds an index column to a database file
Arguments:
db_file = the database file name
field = the name of the index field to add
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'index'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-index',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=field)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Add count and write updated row
rec.update({field: rec_count})
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args):
"""
Deletes entire fields from a database file
Arguments:
db_file = the database file name
fields = a list of fields to drop
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-drop',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write row
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args):
"""
Adds an index column to a database file
Arguments:
db_file = the database file name
field = the name of the index field to add
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'index'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=field)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Add count and write updated row
rec.update({field:rec_count})
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args):
"""
Deletes entire fields from a database file
Arguments:
db_file = the database file name
fields = a list of fields to drop
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write row
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def dropDbFile(db_file, fields, out_file=None, out_args=default_out_args):
"""
Deletes entire fields from a database file
Arguments:
db_file : the database file name.
fields : a list of fields to drop.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
__, __, out_args['out_type'] = splitName(db_file)
# Exclude dropped field from output
out_fields = [f for f in db_iter.fields if f not in fields]
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-drop',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Write row
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def addDbFile(db_file,
fields,
values,
out_file=None,
out_args=default_out_args):
"""
Adds field and value pairs to a database file
Arguments:
db_file : the database file name.
fields : a list of fields to add.
values : a list of values to assign to all rows of each field.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open inut
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
__, __, out_args['out_type'] = splitName(db_file)
# Add fields
out_fields = list(db_iter.fields)
out_fields.extend(fields)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-add',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Define fields and values to append
add_dict = {
k: v
for k, v in zip(fields, values) if k not in db_iter.fields
}
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Write updated row
rec.update(add_dict)
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def convertToFasta(db_file, id_field=default_id_field, seq_field=default_seq_field,
meta_fields=None, out_file=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file : the database file name.
id_field : the field containing identifiers.
seq_field : the field containing sequences.
meta_fields : a list of fields to add to sequence annotations.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ConvertDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open input
out_type = 'fasta'
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
result_count = countDbFile(db_file)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_type)
# Iterate over records
start_time = time()
rec_count, pass_count, fail_count = 0, 0, 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Get SeqRecord
seq = buildSeqRecord(rec, id_field, seq_field, meta_fields)
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, out_type)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ConvertDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def collectPairQueue(alive, result_queue, collect_queue, result_count,
seq_file_1, seq_file_2, out_args):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue = a multiprocessing.Queue holding worker results
collect_queue = a multiprocessing.Queue holding collector return values
result_count = the number of expected assembled sequences
seq_file_1 = the first sequence file name
seq_file_2 = the second sequence file name
out_args = common output argument dictionary from parseCommonArgs
Returns:
None
(adds a dictionary of {log: log object, out_files: output file names} to collect_queue)
"""
try:
# Count records and define output format
out_type = getFileType(seq_file_1) if out_args['out_type'] is None \
else out_args['out_type']
# Defined valid assembly output handle
pass_handle = getOutputHandle(seq_file_1,
'assemble-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
# Defined failed assembly output handles
if out_args['failed']:
# Define output name
if out_args['out_name'] is None:
out_name_1 = out_name_2 = None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
fail_handle_1 = getOutputHandle(seq_file_1,
'assemble-fail',
out_dir=out_args['out_dir'],
out_name=out_name_1,
out_type=out_type)
fail_handle_2 = getOutputHandle(seq_file_2,
'assemble-fail',
out_dir=out_args['out_dir'],
out_name=out_name_2,
out_type=out_type)
else:
fail_handle_1 = None
fail_handle_2 = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
iter_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(iter_count, result_count, 0.05, start_time)
# Update counts for iteration
iter_count += 1
# Write log
printLog(result.log, handle=log_handle)
# Write assembled sequences
if result:
pass_count += 1
SeqIO.write(result.results, pass_handle, out_type)
else:
fail_count += 1
if fail_handle_1 is not None and fail_handle_2 is not None:
SeqIO.write(result.data[0], fail_handle_1, out_type)
SeqIO.write(result.data[1], fail_handle_2, out_type)
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(iter_count, result_count, 0.05, start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['PAIRS'] = iter_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
# Close file handles
pass_handle.close()
if fail_handle_1 is not None: fail_handle_1.close()
if fail_handle_2 is not None: fail_handle_2.close()
if log_handle is not None: log_handle.close()
except:
alive.value = False
raise
return None
def selectDbFile(db_file, fields, values, logic='any', regex=False,
out_args=default_out_args):
"""
Selects records from a database file
Arguments:
db_file = the database file name
fields = a list of fields to check for selection criteria
values = a list of values defining selection targets
logic = one of 'any' or 'all' defining whether one or all fields must have a match.
regex = if False do exact full string matches; if True allow partial regex matches.
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define string match function
if regex:
def _match_func(x, patterns): return any([re.search(p, x) for p in patterns])
else:
def _match_func(x, patterns): return x in patterns
# Define logic function
if logic == 'any':
_logic_func = any
elif logic == 'all':
_logic_func = all
# Print console log
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'select'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
log['REGEX'] =regex
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Check for selection values in all fields
select = _logic_func([_match_func(rec.get(f, False), values) for f in fields])
# Write sequences
if select:
pass_count += 1
pass_writer.writerow(rec)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['SELECTED'] = pass_count
log['DISCARDED'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectDbQueue(alive, result_queue, collect_queue, db_file, task_label, out_args,
add_fields=None):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive : multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue : multiprocessing.Queue holding worker results
collect_queue : multiprocessing.Queue to store collector return values
db_file : Database file name
task_label : Task label used to tag the output files
out_args : Common output argument dictionary from parseCommonArgs
add_fields : List of fields added to the writer not present in the in_file;
if None do not add fields
Returns:
None : Adds a dictionary with key value pairs to collect_queue containing
'log' defining a log object,
'out_files' defining the output file names
"""
try:
result_count = countDbFile(db_file)
# Define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined valid alignment output handle
pass_handle = getOutputHandle(db_file,
'%s-pass' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
# Defined failed alignment output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'%s-fail' % task_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
set_count = rec_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(pass_count, result_count, 0.05, start_time)
# Update counts for current iteration
set_count += 1
rec_count += result.data_count
# Write log
printLog(result.log, handle=log_handle)
# Write alignments
if result:
pass_count += result.data_count
for rec in result.results:
pass_writer.writerow(rec.toDict())
else:
fail_count += result.data_count
if fail_handle is not None:
for rec in result.data:
pass_writer.writerow(rec.toDict())
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(pass_count, result_count, 0.05, start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GROUPS'] = set_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
except:
alive.value = False
raise
return None
def mergeDbFiles(db_files,
drop=False,
out_file=None,
out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_files : list of database file names.
drop : if True drop columns not present in all files.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'merge'
log['FILES'] = ','.join([os.path.basename(f) for f in db_files])
log['DROP'] = drop
printLog(log)
# Open input
db_handles = [open(f, 'rt') for f in db_files]
db_iters = [TSVReader(x) for x in db_handles]
result_count = sum([countDbFile(f) for f in db_files])
# Define output fields
field_list = [x.fields for x in db_iters]
if drop:
field_set = set.intersection(*map(set, field_list))
else:
field_set = set.union(*map(set, field_list))
field_order = OrderedDict([(f, None) for f in chain(*field_list)])
out_fields = [f for f in field_order if f in field_set]
# Open output file
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
__, __, out_args['out_type'] = splitName(db_files[0])
pass_handle = getOutputHandle(db_files[0],
out_label='parse-merge',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Iterate over records
start_time = time()
rec_count = 0
for db in db_iters:
for rec in db:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Write records
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
for x in db_handles:
x.close()
return pass_handle.name
def downsizeSeqFile(seq_file, max_count, out_args=default_out_args):
"""
Splits a FASTA/FASTQ file into segments with a limited number of records
Arguments:
seq_file : filename of the FASTA file to split
max_count : number of records in each output file
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'count'
log['FILE'] = os.path.basename(seq_file)
log['MAX_COUNT'] = max_count
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Determine total numbers of records
rec_count = countSeqFile(seq_file)
# Loop through iterator writing each record and opening new output handle as needed
start_time = time()
seq_count, part_num = 0, 1
out_handle = getOutputHandle(seq_file,
'part%06i' % part_num,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files = [out_handle.name]
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
# Update count
seq_count += 1
# Write records
SeqIO.write(seq, out_handle, out_args['out_type'])
# Break if total records reached to avoid extra empty file
if seq_count == rec_count:
break
# Open new file if needed
if seq_count % max_count == 0:
out_handle.close()
part_num += 1
out_handle = getOutputHandle(seq_file,
'part%06i' % part_num,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files.append(out_handle.name)
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, f in enumerate(out_files):
log['OUTPUT%i' % (i + 1)] = os.path.basename(f)
log['SEQUENCES'] = rec_count
log['PARTS'] = len(out_files)
log['END'] = 'SplitSeq'
printLog(log)
# Close file handles
out_handle.close()
return out_files
def groupSeqFile(seq_file, field, threshold=None, out_args=default_out_args):
"""
Divides a sequence file into segments by description tags
Arguments:
seq_file : filename of the sequence file to split
field : The annotation field to split seq_file by
threshold : The numerical threshold for group sequences by;
if None treat field as textual
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'group'
log['FILE'] = os.path.basename(seq_file)
log['FIELD'] = field
log['THRESHOLD'] = threshold
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Determine total numbers of records
rec_count = countSeqFile(seq_file)
# Process sequences
start_time = time()
seq_count = 0
if threshold is None:
# Sort records into files based on textual field
# Create set of unique field tags
temp_iter = readSeqFile(seq_file)
tag_list = getAnnotationValues(temp_iter,
field,
unique=True,
delimiter=out_args['delimiter'])
if sys.platform != 'win32':
import resource
# Increase open file handle limit if needed
file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
file_count = len(tag_list) + 256
if file_limit < file_count and file_count <= 8192:
#print file_limit, file_count
resource.setrlimit(resource.RLIMIT_NOFILE,
(file_count, file_count))
elif file_count > 8192:
e = '''OS file limit would need to be set to %i.
If you are sure you want to do this, then increase the
file limit in the OS (via ulimit) and rerun this tool.
''' % file_count
printError(dedent(e))
# Create output handles
# out_label = '%s=%s' % (field, tag)
handles_dict = {
tag: getOutputHandle(seq_file,
'%s-%s' % (field, tag),
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
for tag in tag_list
}
# Iterate over sequences
for seq in seq_iter:
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write sequences
tag = parseAnnotation(seq.description,
delimiter=out_args['delimiter'])[field]
SeqIO.write(seq, handles_dict[tag], out_args['out_type'])
else:
# Sort records into files based on numeric threshold
threshold = float(threshold)
# Create output handles
handles_dict = {
'under':
getOutputHandle(seq_file,
'under-%.1g' % threshold,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type']),
'atleast':
getOutputHandle(seq_file,
'atleast-%.1g' % threshold,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
}
# Iterate over sequences
for seq in seq_iter:
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write sequences
tag = parseAnnotation(seq.description,
delimiter=out_args['delimiter'])[field]
tag = 'under' if float(tag) < threshold else 'atleast'
SeqIO.write(seq, handles_dict[tag], out_args['out_type'])
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['SEQUENCES'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'SplitSeq'
printLog(log)
# Close output file handles
for k in handles_dict:
handles_dict[k].close()
return [handles_dict[k].name for k in handles_dict]
def collectPairQueue(alive,
result_queue,
collect_queue,
seq_file_1,
seq_file_2,
label,
out_file=None,
out_args=default_out_args):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive : a multiprocessing.Value boolean controlling whether processing
continues; when False function returns.
result_queue : a multiprocessing.Queue holding worker results.
collect_queue : a multiprocessing.Queue holding collector return values.
seq_file_1 : the first sequence file name.
seq_file_2 : the second sequence file name.
label : task label used to tag the output files.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
None: adds a dictionary of {log: log object, out_files: output file names} to collect_queue.
"""
# Define output format
out_type = getFileType(seq_file_1) if out_args['out_type'] is None \
else out_args['out_type']
# Define output names
if out_args['out_name'] is None:
out_name_1, out_name_2 = None, None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
# Wrapper for opening handles and writers
def _open(x, in_file, out_name, out_file=out_file):
if out_file is not None and x == 'pass':
handle = open(out_file, 'w')
else:
handle = getOutputHandle(in_file,
out_label='%s-%s' % (label, x),
out_dir=out_args['out_dir'],
out_name=out_name,
out_type=out_type)
return handle
try:
# Count input size
result_count = countSeqFile(seq_file_1)
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Initialize file handles
pass_handle, fail_handle_1, fail_handle_2 = None, None, None
# Iterator over results queue until sentinel object reached
start_time = time()
iter_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty():
continue
else:
result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(iter_count,
result_count,
0.05,
start_time=start_time)
# Update counts for iteration
iter_count += 1
# Write log
printLog(result.log, handle=log_handle)
# Write assembled sequences
if result:
pass_count += 1
try:
SeqIO.write(result.results, pass_handle, out_type)
except AttributeError:
# Open pass file
pass_handle = _open('pass', seq_file_1,
out_args['out_name'])
SeqIO.write(result.results, pass_handle, out_type)
else:
fail_count += 1
if out_args['failed']:
try:
SeqIO.write(result.data[0], fail_handle_1, out_type)
SeqIO.write(result.data[1], fail_handle_2, out_type)
except AttributeError:
# Open fail file
fail_handle_1 = _open('fail', seq_file_1, out_name_1)
fail_handle_2 = _open('fail', seq_file_2, out_name_2)
SeqIO.write(result.data[0], fail_handle_1, out_type)
SeqIO.write(result.data[1], fail_handle_2, out_type)
else:
sys.stderr.write('PID %s> Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(iter_count, result_count, 0.05, start_time=start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(
pass_handle.name) if pass_handle is not None else None
log['PAIRS'] = iter_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
# Close file handles and generate return data
# collect_dict = {'log': log, 'pass': None, 'fail': None}
collect_dict = {'log': log, 'out_files': []}
if pass_handle is not None:
# collect_dict['pass'] = pass_handle.name
collect_dict['out_files'].append(pass_handle.name)
pass_handle.close()
if fail_handle_1 is not None:
# collect_dict['fail'] = fail_handle.name
collect_dict['out_files'].append(fail_handle_1.name)
fail_handle_1.close()
if fail_handle_2 is not None:
# collect_dict['fail'] = fail_handle.name
collect_dict['out_files'].append(fail_handle_2.name)
fail_handle_2.close()
if log_handle is not None:
log_handle.close()
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
def collectSeqQueue(alive,
result_queue,
collect_queue,
seq_file,
label,
index_field=None,
out_file=None,
out_args=default_out_args):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive : a multiprocessing.Value boolean controlling whether processing
continues; when False function returns.
result_queue : Multiprocessing.Queue holding worker results.
collect_queue : Multiprocessing.Queue to store collector return values.
seq_file : sample sequence file name.
label : task label used to tag the output files.
out_file : output file name. Automatically generated from the input file if None.
out_args : Common output argument dictionary from parseCommonArgs.
index_field : Field defining set membership for sequence sets
if None data queue contained individual records.
Returns:
None: Adds a dictionary with key value pairs to collect_queue containing
'log' defining a log object,
'out_files' defining the output file names
"""
# Define output format
out_type = getFileType(seq_file) if out_args['out_type'] is None \
else out_args['out_type']
# Wrapper for opening handles and writers
def _open(x, label=label, out_file=out_file):
if out_file is not None and x == 'pass':
handle = open(out_file, 'w')
else:
handle = getOutputHandle(seq_file,
out_label='%s-%s' % (label, x),
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
return handle
try:
# Count records
if index_field is None:
result_count = countSeqFile(seq_file)
else:
result_count = countSeqSets(seq_file, index_field,
out_args['delimiter'])
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Initialize output handles
pass_handle, fail_handle = None, None
# Iterator over results queue until sentinel object reached
start_time = time()
set_count = seq_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(set_count, result_count, 0.05, start_time=start_time)
# Update counts for current iteration
set_count += 1
seq_count += result.data_count
# Write log
printLog(result.log, handle=log_handle)
# Write records
if result:
pass_count += 1
try:
SeqIO.write(result.results, pass_handle, out_type)
except AttributeError:
# Open pass file
pass_handle = _open('pass')
SeqIO.write(result.results, pass_handle, out_type)
else:
fail_count += 1
if out_args['failed']:
try:
SeqIO.write(result.data, fail_handle, out_type)
except AttributeError:
# Open fail file
fail_handle = _open('fail')
SeqIO.write(result.data, fail_handle, out_type)
else:
sys.stderr.write('PID %s> Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(set_count, result_count, 0.05, start_time=start_time)
# Update return values
log = OrderedDict()
log['OUTPUT'] = os.path.basename(
pass_handle.name) if pass_handle is not None else None
log['SEQUENCES'] = seq_count
if index_field is not None:
log['SETS'] = set_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
# Close file handles and generate return data
#collect_dict = {'log': log, 'pass': None, 'fail': None}
collect_dict = {'log': log, 'out_files': []}
if pass_handle is not None:
#collect_dict['pass'] = pass_handle.name
collect_dict['out_files'].append(pass_handle.name)
pass_handle.close()
if fail_handle is not None:
#collect_dict['fail'] = fail_handle.name
collect_dict['out_files'].append(fail_handle.name)
fail_handle.close()
if log_handle is not None:
log_handle.close()
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
def modifyHeaders(seq_file, modify_func, modify_args, out_file=None, out_args=default_out_args):
"""
Modifies sequence headers
Arguments:
seq_file : the sequence file name.
modify_func : the function defining the modification operation.
modify_args : a dictionary of arguments to pass to modify_func.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: output file name.
"""
# Define subcommand label dictionary
cmd_dict = {addHeader: 'add',
copyHeader: 'copy',
collapseHeader: 'collapse',
deleteHeader: 'delete',
expandHeader: 'expand',
renameHeader: 'rename'}
# Print parameter info
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__)
log['FILE'] = os.path.basename(seq_file)
for k in sorted(modify_args):
v = modify_args[k]
log[k.upper()] = ','.join(v) if isinstance(v, list) else v
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(seq_file,
'reheader',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Count records
result_count = countSeqFile(seq_file)
# Iterate over sequences
start_time = time()
seq_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time=start_time)
#Update counts
seq_count += 1
# Modify header
header = parseAnnotation(seq.description, delimiter=out_args['delimiter'])
header = modify_func(header, delimiter=out_args['delimiter'], **modify_args)
# Write new sequence
seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, out_handle, out_args['out_type'])
# print counts
printProgress(seq_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def tableHeaders(seq_file, fields, out_file=None, out_args=default_out_args):
"""
Builds a table of sequence header annotations
Arguments:
seq_file : the sequence file name.
fields : the list of fields to output.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str: output table file name
"""
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open file handles
seq_iter = readSeqFile(seq_file)
if out_file is not None:
out_handle = open(out_file, 'w')
else:
out_handle = getOutputHandle(seq_file,
'headers',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Count records
result_count = countSeqFile(seq_file)
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time=start_time)
# Get annotations
seq_count += 1
ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter'])
# Write records
if ann:
pass_count += 1
out_writer.writerow(ann)
else:
fail_count += 1
# Print counts
printProgress(seq_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def renameDbFile(db_file,
fields,
names,
out_file=None,
out_args=default_out_args):
"""
Renames fields in a database file
Arguments:
db_file : the database file name.
fields : a list of fields to rename.
values : a list of new names for fields.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'rename'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['NAMES'] = ','.join(names)
printLog(log)
# Open file handles
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
__, __, out_args['out_type'] = splitName(db_file)
# Get header and rename fields
out_fields = list(db_iter.fields)
for f, n in zip(fields, names):
i = out_fields.index(f)
out_fields[i] = n
# Open writer
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-rename',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# TODO: repeating renaming is unnecessary.
# Rename fields
for f, n in zip(fields, names):
rec[n] = rec.pop(f)
# Write
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def sortDbFile(db_file,
field,
numeric=False,
descend=False,
out_file=None,
out_args=default_out_args):
"""
Sorts records by values in an annotation field
Arguments:
db_file : the database filename
field : the field name to sort by
numeric : if True sort field numerically;
if False sort field alphabetically
descend : if True sort in descending order;
if False sort in ascending order
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs
Returns:
str : output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
out_fields = db_iter.fields
__, __, out_args['out_type'] = splitName(db_file)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-sort',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Store all records in a dictionary
start_time = time()
printMessage("Indexing: Running", start_time=start_time)
db_dict = {i: r for i, r in enumerate(db_iter)}
result_count = len(db_dict)
# Sort db_dict by field values
tag_dict = {k: v[field] for k, v in db_dict.items()}
if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend)
printMessage("Indexing: Done", start_time=start_time, end=True)
# Iterate over records
start_time = time()
rec_count = 0
for key in sorted_keys:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Write records
pass_writer.writeDict(db_dict[key])
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def sortSeqFile(seq_file,
field,
numeric=False,
max_count=None,
out_args=default_out_args):
"""
Sorts a sequence file by annotation fields
Arguments:
seq_file : filename of the sequence file to split
field : position of field in sequence description to split by
numeric : if True sort field numerically;
if False sort field alphabetically
max_count : maximum number of records in each output file
if None do not create multiple files
out_args : common output argument dictionary from parseCommonArgs
Returns:
list: output file names
"""
log = OrderedDict()
log['START'] = 'SplitSeq'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(seq_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
log['MAX_COUNT'] = max_count
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_dict = readSeqFile(seq_file, index=True)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Get annotations and sort seq_dict by annotation values
tag_dict = {
k: parseAnnotation(seq_dict[k].description,
delimiter=out_args['delimiter'])[field]
for k in seq_dict
}
if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get)
# Determine total numbers of records
rec_count = len(seq_dict)
if max_count >= rec_count: max_count = None
# Open initial output file handles
file_count = 1
if max_count is None: out_label = 'sorted'
else: out_label = 'sorted-part%06i' % file_count
out_handle = getOutputHandle(seq_file,
out_label,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
out_files = [out_handle.name]
# Loop through sorted sequence dictionary keys
start_time = time()
last_tag = None
saved_keys = []
seq_count = chunk_count = 0
for key in sorted_keys:
# Print progress for previous iteration and update count
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
seq_count += 1
# Write saved group of sequences when tag changes
if last_tag is not None and tag_dict[key] != last_tag:
# Open new output file if needed
if max_count is not None and chunk_count + len(
saved_keys) > max_count:
# Update partition counts
file_count += 1
chunk_count = 0
# Open new file handle
out_handle.close()
out_handle = getOutputHandle(seq_file,
'sorted-part%06i' % file_count,
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
# Append output file name to out_files
out_files.append(out_handle.name)
# Write saved sequences
for k in saved_keys:
chunk_count += 1
SeqIO.write(seq_dict[k], out_handle, out_args['out_type'])
# Reset saved keys to current key only
saved_keys = [key]
else:
# Update list of saved keys if tag is unchanged
saved_keys.append(key)
# Check if total records reached, write all saved keys, and exit loop
if seq_count == rec_count:
for k in saved_keys:
chunk_count += 1
SeqIO.write(seq_dict[k], out_handle, out_args['out_type'])
out_handle.close()
break
# Update tag tracker
last_tag = tag_dict[key]
# Print log
printProgress(seq_count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, f in enumerate(out_files):
log['OUTPUT%i' % (i + 1)] = os.path.basename(f)
log['SEQUENCES'] = seq_count
log['PARTS'] = len(out_files)
log['END'] = 'SplitSeq'
printLog(log)
# Close file handles
out_handle.close()
return out_files
def collectQueue(alive, result_queue, collect_queue, db_file, out_args, cluster_func=None, cluster_args={}):
"""
Assembles results from a queue of individual sequence results and manages log/file I/O
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing continues
if False exit process
result_queue = a multiprocessing.Queue holding processQueue results
collect_queue = a multiprocessing.Queue to store collector return values
db_file = the input database file name
out_args = common output argument dictionary from parseCommonArgs
cluster_func = the function to call for carrying out clustering on distance matrix
cluster_args = a dictionary of arguments to pass to cluster_func
Returns:
None
(adds 'log' and 'out_files' to collect_dict)
"""
# Open output files
try:
# Count records and define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
result_count = countDbFile(db_file)
# Defined successful output handle
pass_handle = getOutputHandle(db_file,
out_label='clone-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE')
# Defined failed alignment output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
out_label='clone-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
fail_writer = None
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
#sys.stderr.write('Exception in collector file opening step\n')
alive.value = False
raise
# Get results from queue and write to files
try:
#print 'START COLLECT', alive.value
# Iterator over results queue until sentinel object reached
start_time = time()
rec_count = clone_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
#print "COLLECT", alive.value, result['id']
# Print progress for previous iteration and update record count
if rec_count == 0:
print('PROGRESS> Assigning clones')
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += len(result.data)
# Write passed and failed records
if result:
for clone in result.results.values():
clone_count += 1
for i, rec in enumerate(clone):
rec.annotations['CLONE'] = clone_count
pass_writer.writerow(rec.toDict())
pass_count += 1
result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction)
else:
for i, rec in enumerate(result.data):
if fail_writer is not None: fail_writer.writerow(rec.toDict())
fail_count += 1
result.log['CLONE0-%i' % (i + 1)] = str(rec.junction)
# Write log
printLog(result.log, handle=log_handle)
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print total counts
printProgress(rec_count, result_count, 0.05, start_time)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
# Update return list
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
except:
#sys.stderr.write('Exception in collector result processing step\n')
alive.value = False
raise
return None
def modifyHeaders(seq_file, modify_func, modify_args, out_args=default_out_args):
"""
Modifies sequence headers
Arguments:
seq_file = the sequence file name
modify_func = the function defining the modification operation
modify_args = a dictionary of arguments to pass to modify_func
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define subcommand label dictionary
cmd_dict = {addHeader: 'add',
copyHeader: 'copy',
collapseHeader: 'collapse',
deleteHeader: 'delete',
expandHeader: 'expand',
renameHeader: 'rename'}
# Print parameter info
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__)
log['FILE'] = os.path.basename(seq_file)
for k in sorted(modify_args):
v = modify_args[k]
log[k.upper()] = ','.join(v) if isinstance(v, list) else v
printLog(log)
# Open file handles
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
# Count records
result_count = countSeqFile(seq_file)
# Iterate over sequences
start_time = time()
seq_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time)
#Update counts
seq_count += 1
# Modify header
header = parseAnnotation(seq.description, delimiter=out_args['delimiter'])
header = modify_func(header, delimiter=out_args['delimiter'], **modify_args)
# Write new sequence
seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, out_handle, out_args['out_type'])
# print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def tableHeaders(seq_file, fields, out_args=default_out_args):
"""
Builds a table of sequence header annotations
Arguments:
seq_file = the sequence file name
fields = the list of fields to output
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output table file name
"""
log = OrderedDict()
log['START'] = 'ParseHeaders'
log['COMMAND'] = 'table'
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open file handles
seq_iter = readSeqFile(seq_file)
out_handle = getOutputHandle(seq_file, out_label='headers', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
# Count records
result_count = countSeqFile(seq_file)
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration
printProgress(seq_count, result_count, 0.05, start_time)
# Get annotations
seq_count += 1
ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter'])
# Write records
if ann:
pass_count += 1
out_writer.writerow(ann)
else:
fail_count += 1
# Print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseHeaders'
printLog(log)
# Close file handles
out_handle.close()
return out_handle.name
def assembleEachGermline(db_file,
repo,
germ_types,
v_field,
seq_field,
out_args=default_out_args):
"""
Write germline sequences to tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types,
str) else ','.join(germ_types)
log['CLONED'] = 'False'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
references = readRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS']
# Create output file handle and Db writer
pass_handle = getOutputHandle(db_file,
'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
fail_writer = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
pass_count = fail_count = 0
# Iterate over rows
for i, row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
result_log, germlines = joinGermline(row,
references,
seq_field=seq_field,
v_field=v_field,
germ_types=germ_types)
# Add germline field(s) to dictionary
if 'full' in germ_types:
row['GERMLINE_' + seq_type] = germlines['full']
if 'dmask' in germ_types:
row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask']
if 'vonly' in germ_types:
row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly']
if 'regions' in germ_types:
row['GERMLINE_REGIONS'] = germlines['regions']
# Write row to pass or fail file
if 'ERROR' in result_log:
fail_count += 1
if fail_writer is not None: fail_writer.writerow(row)
else:
result_log['SEQUENCE'] = row[seq_field]
result_log['GERMLINE'] = germlines['full']
result_log['REGIONS'] = germlines['regions']
pass_count += 1
pass_writer.writerow(row)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i + 1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def sortDbFile(db_file, field, numeric=False, descend=False,
out_args=default_out_args):
"""
Sorts records by values in an annotation field
Arguments:
db_file = the database filename
field = the field name to sort by
numeric = if True sort field numerically;
if False sort field alphabetically
descend = if True sort in descending order;
if False sort in ascending order
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'sort'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUMERIC'] = numeric
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Store all records in a dictionary
start_time = time()
printMessage("Indexing: Running", start_time=start_time)
db_dict = {i:r for i, r in enumerate(db_iter)}
result_count = len(db_dict)
# Sort db_dict by field values
tag_dict = {k:v[field] for k, v in db_dict.items()}
if numeric: tag_dict = {k:float(v or 0) for k, v in tag_dict.items()}
sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend)
printMessage("Indexing: Done", start_time=start_time, end=True)
# Iterate over records
start_time = time()
rec_count = 0
for key in sorted_keys:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write records
pass_writer.writerow(db_dict[key])
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def assembleCloneGermline(db_file,
repo,
seq_field=default_seq_field,
v_field=default_v_field,
germ_types=default_germ_types,
out_args=default_out_args):
"""
Assemble one germline sequence for each clone in a tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types,
str) else ','.join(germ_types)
log['CLONED'] = 'True'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
references = readRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types:
add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS']
add_fields += ['GERMLINE_V_CALL']
add_fields += ['GERMLINE_D_CALL']
add_fields += ['GERMLINE_J_CALL']
# Create output file handle and Db writer
writers = {}
pass_handle = getOutputHandle(db_file,
'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
writers['fail'] = getDbWriter(fail_handle,
db_file,
add_fields=add_fields)
else:
fail_handle = None
writers['fail'] = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
counts = {}
clone_count = counts['pass'] = counts['fail'] = 0
# Iterate over rows
clone = 'initial'
clone_dict = OrderedDict()
for i, row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
# Clone isn't over yet
if row.get('CLONE', '') == clone:
clone_dict[i] = row
# Clone just finished
elif clone_dict:
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, references,
germ_types, v_field, seq_field,
counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Now deal with current row (first of next clone)
clone = row['CLONE']
clone_dict = OrderedDict([(i, row)])
# Last case is only for first row of file
else:
clone = row['CLONE']
clone_dict = OrderedDict([(i, row)])
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, references, germ_types,
v_field, seq_field, counts, writers,
out_args)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i + 1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = counts['pass']
log['FAIL'] = counts['fail']
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def convertToBaseline(db_file, id_field=default_id_field, seq_field=default_seq_field,
germ_field=default_germ_field, cluster_field=None,
meta_fields=None, out_file=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file : the database file name.
id_field : the field containing identifiers.
seq_field : the field containing sample sequences.
germ_field : the field containing germline sequences.
cluster_field : the field containing clonal groupings;
if None write the germline for each record.
meta_fields : a list of fields to add to sequence annotations.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name
"""
log = OrderedDict()
log['START'] = 'ConvertDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
log['GERM_FIELD'] = germ_field
log['CLUSTER_FIELD'] = cluster_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
result_count = countDbFile(db_file)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='clip')
# Iterate over records
start_time = time()
rec_count, germ_count, pass_count, fail_count = 0, 0, 0, 0
cluster_last = None
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Update cluster ID
cluster = rec.get(cluster_field, None)
# Get germline SeqRecord when needed
if cluster_field is None:
germ = buildSeqRecord(rec, id_field, germ_field, meta_fields)
germ.id = '>' + germ.id
elif cluster != cluster_last:
germ = buildSeqRecord(rec, cluster_field, germ_field)
germ.id = '>' + germ.id
else:
germ = None
# Get read SeqRecord
seq = buildSeqRecord(rec, id_field, seq_field, meta_fields)
# Write germline
if germ is not None:
germ_count += 1
SeqIO.write(germ, pass_handle, 'fasta')
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, 'fasta')
else:
fail_count += 1
# Set last cluster ID
cluster_last = cluster
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GERMLINES'] = germ_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ConvertDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def writeDb(db_gen, file_prefix, total_count, id_dict={}, no_parse=True,
score_fields=False, region_fields=False, out_args=default_out_args):
"""
Writes tab-delimited database file in output directory
Arguments:
db_gen = a generator of IgRecord objects containing alignment data
file_prefix = directory and prefix for CLIP tab-delim file
total_count = number of records (for progress bar)
id_dict = a dictionary of {IMGT ID: full seq description}
no_parse = if ID is to be parsed for pRESTO output with default delimiters
score_fields = if True add alignment score fields to output file
region_fields = if True add FWR and CDR region fields to output file
out_args = common output argument dictionary from parseCommonArgs
Returns:
None
"""
pass_file = "%s_db-pass.tab" % file_prefix
fail_file = "%s_db-fail.tab" % file_prefix
ordered_fields = ['SEQUENCE_ID',
'SEQUENCE_INPUT',
'FUNCTIONAL',
'IN_FRAME',
'STOP',
'MUTATED_INVARIANT',
'INDELS',
'V_CALL',
'D_CALL',
'J_CALL',
'SEQUENCE_VDJ',
'SEQUENCE_IMGT',
'V_SEQ_START',
'V_SEQ_LENGTH',
'V_GERM_START_VDJ',
'V_GERM_LENGTH_VDJ',
'V_GERM_START_IMGT',
'V_GERM_LENGTH_IMGT',
'N1_LENGTH',
'D_SEQ_START',
'D_SEQ_LENGTH',
'D_GERM_START',
'D_GERM_LENGTH',
'N2_LENGTH',
'J_SEQ_START',
'J_SEQ_LENGTH',
'J_GERM_START',
'J_GERM_LENGTH',
'JUNCTION_LENGTH',
'JUNCTION']
if score_fields:
ordered_fields.extend(['V_SCORE',
'V_IDENTITY',
'V_EVALUE',
'V_BTOP',
'J_SCORE',
'J_IDENTITY',
'J_EVALUE',
'J_BTOP'])
if region_fields:
ordered_fields.extend(['FWR1_IMGT', 'FWR2_IMGT', 'FWR3_IMGT', 'FWR4_IMGT',
'CDR1_IMGT', 'CDR2_IMGT', 'CDR3_IMGT'])
# TODO: This is not the best approach. should pass in output fields.
# Initiate passed handle
pass_handle = None
# Open failed file
if out_args['failed']:
fail_handle = open(fail_file, 'wt')
fail_writer = getDbWriter(fail_handle, add_fields=['SEQUENCE_ID', 'SEQUENCE_INPUT'])
else:
fail_handle = None
fail_writer = None
# Initialize counters and file
pass_writer = None
start_time = time()
rec_count = pass_count = fail_count = 0
for record in db_gen:
#printProgress(i + (total_count/2 if id_dict else 0), total_count, 0.05, start_time)
printProgress(rec_count, total_count, 0.05, start_time)
rec_count += 1
# Count pass or fail
if (record.v_call == 'None' and record.j_call == 'None') or \
record.functional is None or \
not record.seq_vdj or \
not record.junction:
# print(record.v_call, record.j_call, record.functional, record.junction)
fail_count += 1
if fail_writer is not None: fail_writer.writerow(record.toDict())
continue
else:
pass_count += 1
# Build sample sequence description
if record.id in id_dict:
record.id = id_dict[record.id]
# Parse sequence description into new columns
if not no_parse:
record.annotations = parseAnnotation(record.id, delimiter=out_args['delimiter'])
record.id = record.annotations['ID']
del record.annotations['ID']
# TODO: This is not the best approach. should pass in output fields.
# If first sequence, use parsed description to create new columns and initialize writer
if pass_writer is None:
if not no_parse: ordered_fields.extend(list(record.annotations.keys()))
pass_handle = open(pass_file, 'wt')
pass_writer = getDbWriter(pass_handle, add_fields=ordered_fields)
# Write row to tab-delim CLIP file
pass_writer.writerow(record.toDict())
# Print log
#printProgress(i+1 + (total_count/2 if id_dict else 0), total_count, 0.05, start_time)
printProgress(rec_count, total_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = pass_file
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'MakeDb'
printLog(log)
if pass_handle is not None: pass_handle.close()
if fail_handle is not None: fail_handle.close()
def convertToGenbank(db_file, inference=None, db_xref=None, molecule=default_molecule,
product=default_product, features=None, c_field=None, label=None,
count_field=None, index_field=None, allow_stop=False,
asis_id=False, asis_calls=False, allele_delim=default_allele_delim,
build_asn=False, asn_template=None, tbl2asn_exec=default_tbl2asn_exec,
format=default_format, out_file=None,
out_args=default_out_args):
"""
Builds GenBank submission fasta and table files
Arguments:
db_file : the database file name.
inference : reference alignment tool.
db_xref : reference database link.
molecule : source molecule (eg, "mRNA", "genomic DNA")
product : Product (protein) name.
features : dictionary of sample features (BioSample attributes) to add to the description of each record.
c_field : column containing the C region gene call.
label : a string to use as a label for the ID. if None do not add a field label.
count_field : field name to populate the AIRR_READ_COUNT note.
index_field : field name to populate the AIRR_CELL_INDEX note.
allow_stop : if True retain records with junctions having stop codons.
asis_id : if True use the original sequence ID for the output IDs.
asis_calls : if True do not parse gene calls for IMGT nomenclature.
allele_delim : delimiter separating the gene name from the allele number when asis_calls=True.
build_asn : if True run tbl2asn on the generated .tbl and .fsa files.
asn_template : template file (.sbt) to pass to tbl2asn.
tbl2asn_exec : name of or path to the tbl2asn executable.
format : input and output format.
out_file : output file name without extension. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
tuple : the output (feature table, fasta) file names.
"""
log = OrderedDict()
log['START'] = 'ConvertDb'
log['COMMAND'] = 'genbank'
log['FILE'] = os.path.basename(db_file)
printLog(log)
# Define format operators
try:
reader, __, schema = getFormatOperators(format)
except ValueError:
printError('Invalid format %s.' % format)
# Open input
db_handle = open(db_file, 'rt')
db_iter = reader(db_handle)
# Check for required columns
try:
required = ['sequence_input',
'v_call', 'd_call', 'j_call',
'v_seq_start', 'd_seq_start', 'j_seq_start']
checkFields(required, db_iter.fields, schema=schema)
except LookupError as e:
printError(e)
# Open output
if out_file is not None:
out_name, __ = os.path.splitext(out_file)
fsa_handle = open('%s.fsa' % out_name, 'w')
tbl_handle = open('%s.tbl' % out_name, 'w')
else:
fsa_handle = getOutputHandle(db_file, out_label='genbank', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='fsa')
tbl_handle = getOutputHandle(db_file, out_label='genbank', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tbl')
# Count records
result_count = countDbFile(db_file)
# Define writer
writer = csv.writer(tbl_handle, delimiter='\t', quoting=csv.QUOTE_NONE)
# Iterate over records
start_time = time()
rec_count, pass_count, fail_count = 0, 0, 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Extract table dictionary
name = None if asis_id else rec_count
seq = makeGenbankSequence(rec, name=name, label=label, count_field=count_field, index_field=index_field,
molecule=molecule, features=features)
tbl = makeGenbankFeatures(rec, start=seq['start'], end=seq['end'], product=product,
db_xref=db_xref, inference=inference, c_field=c_field,
allow_stop=allow_stop, asis_calls=asis_calls, allele_delim=allele_delim)
if tbl is not None:
pass_count +=1
# Write table
writer.writerow(['>Features', seq['record'].id])
for feature, qualifiers in tbl.items():
writer.writerow(feature)
if qualifiers:
for x in qualifiers:
writer.writerow(list(chain(['', '', ''], x)))
# Write sequence
SeqIO.write(seq['record'], fsa_handle, 'fasta')
else:
fail_count += 1
# Final progress bar
printProgress(rec_count, result_count, 0.05, start_time=start_time)
# Run tbl2asn
if build_asn:
start_time = time()
printMessage('Running tbl2asn', start_time=start_time, width=25)
result = runASN(fsa_handle.name, template=asn_template, exec=tbl2asn_exec)
printMessage('Done', start_time=start_time, end=True, width=25)
# Print ending console log
log = OrderedDict()
log['OUTPUT_TBL'] = os.path.basename(tbl_handle.name)
log['OUTPUT_FSA'] = os.path.basename(fsa_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ConvertDb'
printLog(log)
# Close file handles
tbl_handle.close()
fsa_handle.close()
db_handle.close()
return (tbl_handle.name, fsa_handle.name)
def convertDbFasta(db_file,
id_field=default_id_field,
seq_field=default_seq_field,
meta_fields=None,
out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sequences
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
out_type = 'fasta'
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='sequences',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Get SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields,
out_args['delimiter'])
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, out_type)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def indexDbFile(db_file,
field=default_index_field,
out_file=None,
out_args=default_out_args):
"""
Adds an index column to a database file
Arguments:
db_file : the database file name.
field : the name of the index field to add.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'index'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
__, __, out_args['out_type'] = splitName(db_file)
# Append index field
out_fields = list(db_iter.fields)
out_fields.append(field)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-index',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Add count and write updated row
rec.update({field: rec_count})
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def addDbFile(db_file, fields, values, out_args=default_out_args):
"""
Adds field and value pairs to a database file
Arguments:
db_file = the database file name
fields = a list of fields to add
values = a list of values to assign to all rows of each field
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'add'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-add',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields)
# Count records
result_count = countDbFile(db_file)
# Define fields and values to append
add_dict = {
k: v
for k, v in zip(fields, values) if k not in db_iter.fieldnames
}
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Write updated row
rec.update(add_dict)
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args):
"""
Divides a tab-delimited database file into segments by description tags
Arguments:
db_file : filename of the tab-delimited database file to split
field : the field name by which to split db_file
num_split : the numerical threshold by which to group sequences;
if None treat field as textual
out_args : common output argument dictionary from parseCommonArgs
Returns:
list : a list of output file names.
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'split'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUM_SPLIT'] = num_split
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
out_fields = db_iter.fields
__, __, out_args['out_type'] = splitName(db_file)
# Determine total numbers of records
rec_count = countDbFile(db_file)
start_time = time()
count = 0
# Sort records into files based on textual field
if num_split is None:
# Create set of unique field tags
with open(db_file, 'rt') as tmp_handle:
tmp_iter = TSVReader(tmp_handle)
tag_list = list(set([row[field] for row in tmp_iter]))
# Forbidden characters in filename and replacements
no_good = {
'\/': 'f',
'\\': 'b',
'?': 'q',
'\%': 'p',
'*': 's',
':': 'c',
'\|': 'pi',
'\"': 'dq',
'\'': 'sq',
'<': 'gt',
'>': 'lt',
' ': '_'
}
# Replace forbidden characters in tag_list
tag_dict = {}
for tag in tag_list:
for c, r in no_good.items():
tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \
if c in tag else tag_dict.get(tag, tag))
# Create output handles
handles_dict = {
tag: getOutputHandle(db_file,
out_label='%s-%s' % (field, label),
out_name=out_args['out_name'],
out_dir=out_args['out_dir'],
out_type=out_args['out_type'])
for tag, label in tag_dict.items()
}
# Create Db writer instances
writers_dict = {
tag: TSVWriter(handles_dict[tag], fields=out_fields)
for tag in tag_dict
}
# Iterate over records
for row in db_iter:
printProgress(count, rec_count, 0.05, start_time=start_time)
count += 1
# Write row to appropriate file
tag = row[field]
writers_dict[tag].writeDict(row)
# Sort records into files based on numeric num_split
else:
num_split = float(num_split)
# Create output handles
handles_dict = {
'under':
getOutputHandle(db_file,
out_label='under-%.1f' % num_split,
out_name=out_args['out_name'],
out_dir=out_args['out_dir'],
out_type=out_args['out_type']),
'atleast':
getOutputHandle(db_file,
out_label='atleast-%.1f' % num_split,
out_name=out_args['out_name'],
out_dir=out_args['out_dir'],
out_type=out_args['out_type'])
}
# Create Db writer instances
writers_dict = {
'under': TSVWriter(handles_dict['under'], fields=out_fields),
'atleast': TSVWriter(handles_dict['atleast'], fields=out_fields)
}
# Iterate over records
for row in db_iter:
printProgress(count, rec_count, 0.05, start_time=start_time)
count += 1
tag = row[field]
tag = 'under' if float(tag) < num_split else 'atleast'
writers_dict[tag].writeDict(row)
# Write log
printProgress(count, rec_count, 0.05, start_time=start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['RECORDS'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'ParseDb'
printLog(log)
# Close output file handles
db_handle.close()
for t in handles_dict:
handles_dict[t].close()
return [handles_dict[t].name for t in handles_dict]
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args):
"""
Write germline sequences to tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types)
log['CLONED'] = 'False'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
repo_dict = getRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
# Create output file handle and Db writer
pass_handle = getOutputHandle(db_file, 'germ-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file, 'germ-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
fail_writer = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
pass_count = fail_count = 0
# Iterate over rows
for i,row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
result_log, germlines = joinGermline(row, repo_dict, germ_types, v_field, seq_field)
# Add germline field(s) to dictionary
if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full']
if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask']
if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly']
# Write row to pass or fail file
if 'ERROR' in result_log:
fail_count += 1
if fail_writer is not None: fail_writer.writerow(row)
else:
result_log['SEQUENCE'] = row[seq_field]
result_log['GERMLINE'] = germlines['full']
result_log['REGIONS'] = germlines['regions']
pass_count += 1
pass_writer.writerow(row)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i+1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def selectDbFile(db_file,
fields,
values,
logic='any',
regex=False,
out_file=None,
out_args=default_out_args):
"""
Selects records from a database file
Arguments:
db_file : the database file name
fields : a list of fields to check for selection criteria
values : a list of values defining selection targets
logic : one of 'any' or 'all' defining whether one or all fields must have a match.
regex : if False do exact full string matches; if True allow partial regex matches.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs
Returns:
str : output file name.
"""
# Define string match function
if regex:
def _match_func(x, patterns):
return any([re.search(p, x) for p in patterns])
else:
def _match_func(x, patterns):
return x in patterns
# Define logic function
if logic == 'any':
_logic_func = any
elif logic == 'all':
_logic_func = all
# Print console log
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'select'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
log['REGEX'] = regex
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
out_fields = db_iter.fields
__, __, out_args['out_type'] = splitName(db_file)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-select',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count, pass_count, fail_count = 0, 0, 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Check for selection values in all fields
select = _logic_func(
[_match_func(rec.get(f, False), values) for f in fields])
# Write sequences
if select:
pass_count += 1
pass_writer.writeDict(rec)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['SELECTED'] = pass_count
log['DISCARDED'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def indexJunctions(db_iter, fields=None, mode='gene', action='first'):
"""
Identifies preclonal groups by V, J and junction length
Arguments:
db_iter = an iterator of IgRecords defined by readDbFile
fields = additional annotation fields to use to group preclones;
if None use only V, J and junction length
mode = specificity of alignment call to use for assigning preclones;
one of ('allele', 'gene')
action = how to handle multiple value fields when assigning preclones;
one of ('first', 'set')
Returns:
a dictionary of {(V, J, junction length):[IgRecords]}
"""
# Define functions for grouping keys
if mode == 'allele' and fields is None:
def _get_key(rec, act):
return (rec.getVAllele(act), rec.getJAllele(act),
None if rec.junction is None else len(rec.junction))
elif mode == 'gene' and fields is None:
def _get_key(rec, act):
return (rec.getVGene(act), rec.getJGene(act),
None if rec.junction is None else len(rec.junction))
elif mode == 'allele' and fields is not None:
def _get_key(rec, act):
vdj = [rec.getVAllele(act), rec.getJAllele(act),
None if rec.junction is None else len(rec.junction)]
ann = [rec.toDict().get(k, None) for k in fields]
return tuple(chain(vdj, ann))
elif mode == 'gene' and fields is not None:
def _get_key(rec, act):
vdj = [rec.getVGene(act), rec.getJGene(act),
None if rec.junction is None else len(rec.junction)]
ann = [rec.toDict().get(k, None) for k in fields]
return tuple(chain(vdj, ann))
start_time = time()
clone_index = {}
rec_count = 0
for rec in db_iter:
key = _get_key(rec, action)
# Print progress
if rec_count == 0:
print('PROGRESS> Grouping sequences')
printProgress(rec_count, step=1000, start_time=start_time)
rec_count += 1
# Assigned passed preclone records to key and failed to index None
if all([k is not None and k != '' for k in key]):
#print key
# TODO: Has much slow. Should have less slow.
if action == 'set':
f_range = list(range(2, 3 + (len(fields) if fields else 0)))
vdj_range = list(range(2))
# Check for any keys that have matching columns and junction length and overlapping genes/alleles
to_remove = []
if len(clone_index) > (1 if None in clone_index else 0) and key not in clone_index:
key = list(key)
for k in clone_index:
if k is not None and all([key[i] == k[i] for i in f_range]):
if all([not set(key[i]).isdisjoint(set(k[i])) for i in vdj_range]):
for i in vdj_range: key[i] = tuple(set(key[i]).union(set(k[i])))
to_remove.append(k)
# Remove original keys, replace with union of all genes/alleles and append values to new key
val = [rec]
val += list(chain(*(clone_index.pop(k) for k in to_remove)))
clone_index[tuple(key)] = clone_index.get(tuple(key),[]) + val
elif action == 'first':
clone_index.setdefault(key, []).append(rec)
else:
clone_index.setdefault(None, []).append(rec)
printProgress(rec_count, step=1000, start_time=start_time, end=True)
return clone_index
def updateDbFile(db_file,
field,
values,
updates,
out_file=None,
out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_file : the database file name.
field : the field to update.
values : a list of values to specifying which rows to update.
updates : a list of values to update each value with.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'update'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['VALUES'] = ','.join(values)
log['UPDATES'] = ','.join(updates)
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = TSVReader(db_handle)
out_fields = db_iter.fields
__, __, out_args['out_type'] = splitName(db_file)
# Open output
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file,
out_label='parse-update',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
pass_writer = TSVWriter(pass_handle, out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count, pass_count = 0, 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Updated values if found
for x, y in zip(values, updates):
if rec[field] == x:
rec[field] = y
pass_count += 1
# Write records
pass_writer.writeDict(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['UPDATED'] = pass_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def convertDbClip(db_file, id_field=default_id_field, seq_field=default_seq_field,
germ_field=default_germ_field, cluster_field=None,
meta_fields=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sample sequences
germ_field = the field containing germline sequences
cluster_field = the field containing clonal groupings
if None write the germline for each record
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
log['GERM_FIELD'] = germ_field
log['CLUSTER_FIELD'] = cluster_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='clip')
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = germ_count = pass_count = fail_count = 0
cluster_last = None
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Update cluster ID
cluster = rec.get(cluster_field, None)
# Get germline SeqRecord when needed
if cluster_field is None:
germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
elif cluster != cluster_last:
germ = getDbSeqRecord(rec, cluster_field, germ_field,
delimiter=out_args['delimiter'])
germ.id = '>' + germ.id
else:
germ = None
# Get read SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields,
delimiter=out_args['delimiter'])
# Write germline
if germ is not None:
germ_count += 1
SeqIO.write(germ, pass_handle, 'fasta')
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, 'fasta')
else:
fail_count += 1
# Set last cluster ID
cluster_last = cluster
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['GERMLINES'] = germ_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def tableLog(record_file, fields, out_args=default_out_args):
"""
Converts a pRESTO log to a table of annotations
Arguments:
record_file = the log file name
fields = the list of fields to output
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output table file name
"""
log = OrderedDict()
log['START'] = 'ParseLog'
log['FILE'] = os.path.basename(record_file)
printLog(log)
# Open file handles
log_handle = open(record_file, 'rU')
out_handle = getOutputHandle(record_file,
'table',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Open csv writer and write header
out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='',
delimiter='\t', fieldnames=fields)
out_writer.writeheader()
# Iterate over log records
start_time = time()
record = ''
rec_count = pass_count = fail_count = 0
for line in log_handle:
if line.strip() == '' and record:
# Print progress for previous iteration
printProgress(rec_count, None, 1e5, start_time)
# Parse record block
rec_count += 1
record_dict = parseLogRecord(record)
# Write records
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Empty record string
record = ''
else:
# Append to record
record += line
else:
# Write final record
if record:
record_dict = parseLogRecord(record)
if any([f in fields for f in record_dict]):
pass_count += 1
out_writer.writerow(record_dict)
elif record_dict:
fail_count += 1
# Print counts
printProgress(rec_count, None, 1e5, start_time, end=True)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(out_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseLog'
printLog(log)
# Close file handles
log_handle.close()
out_handle.close()
return log_handle.name
def findUniqueSeq(uniq_dict, search_keys, seq_dict, max_missing=default_max_missing,
uniq_fields=None, copy_fields=None, max_field=None, min_field=None,
inner=False, delimiter=default_delimiter):
"""
Finds unique sequences
Arguments:
uniq_dict = a dictionary of unique sequences generated by findUniqueSeq()
search_keys = a list containing the subset of dictionary keys to be checked
seq_dict = a SeqRecords dictionary generated by SeqIO.index()
max_missing = the number of missing characters to allow in a unique sequences
uniq_fields = a list of annotations that define a sequence as unique if they differ
copy_fields = a list of annotations to copy into unique sequence annotations
max_field = a numeric field whose maximum value determines the retained sequence
min_field = a numeric field whose minimum value determines the retained sequence
inner = if True exclude consecutive outer ambiguous characters from iterations and matching
delimiter = description field delimiter
Returns:
a tuple of (uniq_dict, search_keys, dup_keys) modified from passed values
"""
# Define local variables
ambig_re = re.compile(r'[\.\-N]')
score = (max_missing > 0)
dup_keys = []
to_remove = []
start_time = time()
result_count = len(search_keys)
print('MISSING> %i' % max_missing)
# Iterate over search keys and update uniq_dict and dup_keys
for idx, key in enumerate(search_keys):
# Print progress of previous iteration
printProgress(idx, result_count, 0.05, start_time)
# Define sequence to process
seq = seq_dict[key]
seq_str = str(seq.seq)
if inner: seq_str = seq_str.strip('.-N')
# Skip processing of ambiguous sequences over max_missing threshold
ambig_count = len(ambig_re.findall(seq_str))
if ambig_count > max_missing: continue
# Parse annotation and define unique identifiers (uid)
if uniq_fields is not None:
ann = parseAnnotation(seq_dict[key].description, uniq_fields, delimiter=delimiter)
uid = tuple(chain([seq_str], list(ann.values())))
else:
uid = (seq_str, None)
# Parse annotation and define copied identifiers (cid)
if copy_fields is not None:
ann = parseAnnotation(seq.description, copy_fields, delimiter=delimiter)
#print ann
#cid = [[a] for a in ann.values()]
cid = [[ann.get(k)] for k in copy_fields]
#print cid
else:
cid = []
# Store new unique sequences and process duplicates
match = findUID(uid, uniq_dict, score)
if match is None:
uniq_dict[uid] = list(chain([seq, 1, ambig_count], cid))
else:
# Updated sequence, count, ambiguous character count, and count sets
dup_key = key
uniq_dict[match][1] += 1
for x, c in enumerate(cid):
uniq_dict[match][3 + x].extend(c)
# Check whether to replace previous unique sequence with current sequence
if ambig_count <= uniq_dict[match][2]:
swap = False
seq_last = uniq_dict[match][0]
if max_field is not None:
swap = float(parseAnnotation(seq.description, delimiter=delimiter)[max_field]) > \
float(parseAnnotation(seq_last.description, delimiter=delimiter)[max_field])
elif min_field is not None:
swap = float(parseAnnotation(seq.description, delimiter=delimiter)[min_field]) > \
float(parseAnnotation(seq_last.description, delimiter=delimiter)[min_field])
# TODO: quality evaluation is a bottleneck
else:
if hasattr(seq, 'letter_annotations') and 'phred_quality' in seq.letter_annotations:
q_this = float(sum(seq.letter_annotations['phred_quality'])) / len(seq)
q_last = float(sum(seq_last.letter_annotations['phred_quality'])) / len(seq_last)
swap = q_this > q_last
# Replace old sequence if criteria passed
if swap:
dup_key = seq_last.id
#uniq_dict[match] = [seq, uniq_dict[match][1], ambig_count]
uniq_dict[match][0] = seq
uniq_dict[match][2] = ambig_count
# Update duplicate list
dup_keys.append(dup_key)
# Mark seq for removal from later steps
to_remove.append(idx)
# Remove matched sequences from search_keys
for j in reversed(to_remove): del search_keys[j]
# Update progress
printProgress(result_count, result_count, 0.05, start_time)
return (uniq_dict, search_keys, dup_keys)
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args, cluster_func, cluster_args):
"""
Assembles results from a queue of individual sequence results and manages log/file I/O
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing continues
if False exit process
result_queue = a multiprocessing.Queue holding processQueue results
collect_queue = a multiprocessing.Queue to store collector return values
db_file = the input database file name
out_args = common output argument dictionary from parseCommonArgs
cluster_func = the function to call for carrying out clustering on distance matrix
cluster_args = a dictionary of arguments to pass to cluster_func
Returns:
None
(adds 'log' and 'out_files' to collect_dict)
"""
# Open output files
try:
# Iterate over Ig records to count and order by junction length
result_count = 0
records = {}
# print 'Reading file...'
db_iter = readDbFile(db_file)
for rec in db_iter:
records[rec.id] = rec
result_count += 1
records = OrderedDict(sorted(list(records.items()), key=lambda i: i[1].junction_length))
# Define empty matrix to store assembled results
dist_mat = np.zeros((result_count,result_count))
# Count records and define output format
out_type = getFileType(db_file) if out_args['out_type'] is None \
else out_args['out_type']
# Defined successful output handle
pass_handle = getOutputHandle(db_file,
out_label='clone-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE')
# Defined failed cloning output handle
if out_args['failed']:
fail_handle = getOutputHandle(db_file,
out_label='clone-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_type)
fail_writer = getDbWriter(fail_handle, db_file)
else:
fail_handle = None
fail_writer = None
# Open log file
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
row_count = rec_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
if row_count == 0:
print('PROGRESS> Assigning clones')
printProgress(row_count, result_count, 0.05, start_time)
# Update counts for iteration
row_count += 1
rec_count += len(result)
# Add result row to distance matrix
if result:
dist_mat[list(range(result_count-len(result),result_count)),result_count-len(result)] = result.results
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Calculate linkage and carry out clustering
# print dist_mat
clusters = cluster_func(dist_mat, **cluster_args) if dist_mat is not None else None
clones = {}
# print clusters
for i, c in enumerate(clusters):
clones.setdefault(c, []).append(records[list(records.keys())[i]])
# Write passed and failed records
clone_count = pass_count = fail_count = 0
if clones:
for clone in clones.values():
clone_count += 1
for i, rec in enumerate(clone):
rec.annotations['CLONE'] = clone_count
pass_writer.writerow(rec.toDict())
pass_count += 1
#result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction)
else:
for i, rec in enumerate(result.data):
fail_writer.writerow(rec.toDict())
fail_count += 1
#result.log['CLONE0-%i' % (i + 1)] = str(rec.junction)
# Print final progress
printProgress(row_count, result_count, 0.05, start_time)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
# Update return list
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
collect_dict = {'log':log, 'out_files': [pass_handle.name]}
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
def assembleCloneGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args):
"""
Assemble one germline sequence for each clone in a tab-delimited database file
Arguments:
db_file = input tab-delimited database file
repo = folder with germline repertoire files
germ_types = types of germline sequences to be output
(full germline, D-region masked, only V-region germline)
v_field = field in which to look for V call
seq_field = field in which to look for sequence
out_args = arguments for output preferences
Returns:
None
"""
# Print parameter info
log = OrderedDict()
log['START'] = 'CreateGermlines'
log['DB_FILE'] = os.path.basename(db_file)
log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types)
log['CLONED'] = 'True'
log['V_FIELD'] = v_field
log['SEQ_FIELD'] = seq_field
printLog(log)
# Get repertoire and open Db reader
repo_dict = getRepo(repo)
reader = readDbFile(db_file, ig=False)
# Exit if V call field does not exist in reader
if v_field not in reader.fieldnames:
sys.exit('Error: V field does not exist in input database file.')
# Define log handle
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
add_fields = []
seq_type = seq_field.split('_')[-1]
if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type]
if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK']
if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION']
# Create output file handle and Db writer
writers = {}
pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields)
if out_args['failed']:
fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_args['out_type'])
writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields)
else:
fail_handle = None
writers['fail'] = None
# Initialize time and total count for progress bar
start_time = time()
rec_count = countDbFile(db_file)
counts = {}
clone_count = counts['pass'] = counts['fail'] = 0
# Iterate over rows
clone = 'initial'
clone_dict = OrderedDict()
for i,row in enumerate(reader):
# Print progress
printProgress(i, rec_count, 0.05, start_time)
# Clone isn't over yet
if row.get('CLONE','') == clone:
clone_dict[row["SEQUENCE_ID"]] = row
# Clone just finished
elif clone_dict:
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types,
v_field, seq_field, counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Now deal with current row (first of next clone)
clone = row['CLONE']
clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)])
# Last case is only for first row of file
else:
clone = row['CLONE']
clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)])
clone_count += 1
result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field,
seq_field, counts, writers, out_args)
printLog(result_log, handle=log_handle)
# Print log
printProgress(i+1, rec_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['CLONES'] = clone_count
log['RECORDS'] = rec_count
log['PASS'] = counts['pass']
log['FAIL'] = counts['fail']
log['END'] = 'CreateGermlines'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
if log_handle is not None: log_handle.close()
def convertHeaders(seq_file, convert_func, convert_args={}, out_args=default_out_args):
"""
Converts sequence headers to the pRESTO format
Arguments:
seq_file = the sequence file name
convert_func = the function used to convert sequence headers
convert_args = a dictionary of arguments to pass to convert_func
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output sequence file name
"""
# Define subcommand label dictionary
cmd_dict = {convertGenericHeader:'generic',
convert454Header:'454',
convertGenbankHeader:'genbank',
convertIlluminaHeader:'illumina',
convertIMGTHeader:'imgt',
convertSRAHeader:'sra'}
log = OrderedDict()
log['START'] = 'ConvertHeaders'
log['COMMAND'] = cmd_dict[convert_func]
log['FILE'] = os.path.basename(seq_file)
printLog(log)
# Open input file
in_type = getFileType(seq_file)
seq_iter = readSeqFile(seq_file)
if out_args['out_type'] is None: out_args['out_type'] = in_type
# Count records
result_count = countSeqFile(seq_file)
# Open output file handles
pass_handle = getOutputHandle(seq_file,
'convert-pass',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
if out_args['failed']:
fail_handle = getOutputHandle(seq_file,
'convert-fail',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type=out_args['out_type'])
else:
fail_handle = None
# Set additional conversion arguments
if convert_func in [convertGenericHeader, convertGenbankHeader]:
convert_args.update({'delimiter':out_args['delimiter']})
# Iterate over sequences
start_time = time()
seq_count = pass_count = fail_count = 0
for seq in seq_iter:
# Print progress for previous iteration and update count
printProgress(seq_count, result_count, 0.05, start_time)
seq_count += 1
# Convert header
header = convert_func(seq.description, **convert_args)
if header is not None:
# Write successfully converted sequences
pass_count += 1
seq.id = seq.name = flattenAnnotation(header, out_args['delimiter'])
seq.description = ''
SeqIO.write(seq, pass_handle, out_args['out_type'])
else:
fail_count += 1
if fail_handle is not None:
# Write successfully unconverted sequences
SeqIO.write(seq, fail_handle, out_args['out_type'])
# Print counts
printProgress(seq_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ConvertHeaders'
printLog(log)
# Close file handles
pass_handle.close()
if fail_handle is not None: fail_handle.close()
return pass_handle.name
def deleteDbFile(db_file,
fields,
values,
logic='any',
regex=False,
out_args=default_out_args):
"""
Deletes records from a database file
Arguments:
db_file = the database file name
fields = a list of fields to check for deletion criteria
values = a list of values defining deletion targets
logic = one of 'any' or 'all' defining whether one or all fields must have a match.
regex = if False do exact full string matches; if True allow partial regex matches.
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
# Define string match function
if regex:
def _match_func(x, patterns):
return any([re.search(p, x) for p in patterns])
else:
def _match_func(x, patterns):
return x in patterns
# Define logic function
if logic == 'any':
_logic_func = any
elif logic == 'all':
_logic_func = all
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'delete'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['VALUES'] = ','.join(values)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-delete',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Check for deletion values in all fields
delete = _logic_func(
[_match_func(rec.get(f, False), values) for f in fields])
# Write sequences
if not delete:
pass_count += 1
pass_writer.writerow(rec)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['KEPT'] = pass_count
log['DELETED'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field,
meta_fields=None, out_args=default_out_args):
"""
Builds fasta files from database records
Arguments:
db_file = the database file name
id_field = the field containing identifiers
seq_field = the field containing sequences
meta_fields = a list of fields to add to sequence annotations
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'fasta'
log['FILE'] = os.path.basename(db_file)
log['ID_FIELD'] = id_field
log['SEQ_FIELD'] = seq_field
if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
printLog(log)
# Open file handles
out_type = 'fasta'
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=out_type)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = fail_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Get SeqRecord
seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter'])
# Write sequences
if seq is not None:
pass_count += 1
SeqIO.write(seq, pass_handle, out_type)
else:
fail_count += 1
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args):
"""
Renames fields in a database file
Arguments:
db_file = the database file name
fields = a list of fields to rename
values = a list of new names for fields
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'rename'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['NAMES'] = ','.join(names)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-rename',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
# Get header and rename fields
header = (readDbFile(db_file, ig=False)).fieldnames
for f, n in zip(fields, names):
i = header.index(f)
header[i] = n
# Open writer and write new header
# TODO: should modify getDbWriter to take a list of fields
pass_writer = csv.DictWriter(pass_handle,
fieldnames=header,
dialect='excel-tab')
pass_writer.writeheader()
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore
# Rename fields
for f, n in zip(fields, names):
rec[n] = rec.pop(f)
# Write
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def updateDbFile(db_file, field, values, updates, out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_file = the database file name
field = the field to update
values = a list of values to specifying which rows to update
updates = a list of values to update each value with
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'update'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['VALUES'] = ','.join(values)
log['UPDATES'] = ','.join(updates)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file,
out_label='parse-update',
out_dir=out_args['out_dir'],
out_name=out_args['out_name'],
out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Updated values if found
for x, y in zip(values, updates):
if rec[field] == x:
rec[field] = y
pass_count += 1
# Write records
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['UPDATED'] = pass_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def pairSeq(seq_file_1, seq_file_2, fields_1=None, fields_2=None,
coord_type=default_coord_type,
out_args=default_out_args):
"""
Generates consensus sequences
Arguments:
seq_file_1 = the file containing the grouped sequences and annotations
seq_file_2 = the file to assign annotations to from seq_file_1
fields_1 = list of annotations in seq_file_1 records to copy to seq_file_2 records;
if None do not copy any annotations
fields_2 = list of annotations in seq_file_2 records to copy to seq_file_1 records;
if None do not copy any annotations
coord_type = the sequence header format
out_args = common output argument dictionary from parseCommonArgs
Returns:
a list of tuples holding successfully paired filenames for (seq_file_1, seq_file_2)
"""
# Define private functions
def _key_func(x):
return getCoordKey(x, coord_type=coord_type, delimiter=out_args['delimiter'])
log = OrderedDict()
log['START'] = 'PairSeq'
log['FILE1'] = os.path.basename(seq_file_1)
log['FILE2'] = os.path.basename(seq_file_2)
log['FIELDS_1'] = ','.join(fields_1) if fields_1 is not None else None
log['FIELDS_2'] = ','.join(fields_2) if fields_2 is not None else None
log['COORD_TYPE'] = coord_type
printLog(log)
# Define output type
if out_args['out_type'] is None:
out_type_1 = getFileType(seq_file_1)
out_type_2 = getFileType(seq_file_2)
else:
out_type_1 = out_type_2 = out_args['out_type']
# Define output name
if out_args['out_name'] is None:
out_name_1 = out_name_2 = None
else:
out_name_1 = '%s-1' % out_args['out_name']
out_name_2 = '%s-2' % out_args['out_name']
# Open and count files
start_time = time()
printMessage("Indexing files", start_time=start_time)
# Index file 1
seq_count_1 = countSeqFile(seq_file_1)
seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func)
# Define file 2 iterator
seq_count_2 = countSeqFile(seq_file_2)
seq_iter_2 = readSeqFile(seq_file_2, index=False)
printMessage("Done", start_time=start_time, end=True)
# Open output file handles
pass_handle_1 = getOutputHandle(seq_file_1, 'pair-pass', out_args['out_dir'],
out_name=out_name_1, out_type=out_type_1)
pass_handle_2 = getOutputHandle(seq_file_2, 'pair-pass', out_args['out_dir'],
out_name=out_name_2, out_type=out_type_2)
if out_args['failed']:
fail_handle_1 = getOutputHandle(seq_file_1, 'pair-fail', out_dir=out_args['out_dir'],
out_name=out_name_1, out_type=out_type_1)
fail_handle_2 = getOutputHandle(seq_file_2, 'pair-fail', out_dir=out_args['out_dir'],
out_name=out_name_2, out_type=out_type_2)
pass_keys = list()
# Iterate over pairs and write to output files
start_time = time()
rec_count = pair_count = 0
for seq_2 in seq_iter_2:
# Print progress for previous iteration
printProgress(rec_count, seq_count_2, 0.05, start_time)
rec_count += 1
# Check for file 2 mate pair in file 1
coord_2 = getCoordKey(seq_2.id, coord_type=coord_type,
delimiter=out_args['delimiter'])
seq_1 = seq_dict_1.get(coord_2, None)
if seq_1 is not None:
# Record paired keys
pair_count += 1
if fields_1 is not None or fields_2 is not None:
ann_1 = parseAnnotation(seq_1.description,
delimiter=out_args['delimiter'])
ann_2 = parseAnnotation(seq_2.description,
delimiter=out_args['delimiter'])
# Prepend annotations from seq_1 to seq_2
if fields_1 is not None:
copy_ann = OrderedDict([(k, v) for k, v in ann_1.items() \
if k in fields_1])
merge_ann = mergeAnnotation(ann_2, copy_ann, prepend=True,
delimiter=out_args['delimiter'])
seq_2.id = flattenAnnotation(merge_ann,
delimiter=out_args['delimiter'])
seq_2.description = ''
# Append annotations from seq_2 to seq_1
if fields_2 is not None:
copy_ann = OrderedDict([(k, v) for k, v in ann_2.items() \
if k in fields_2])
merge_ann = mergeAnnotation(ann_1, copy_ann, prepend=False,
delimiter=out_args['delimiter'])
seq_1.id = flattenAnnotation(merge_ann,
delimiter=out_args['delimiter'])
seq_1.description = ''
# Write paired records
SeqIO.write(seq_1, pass_handle_1, out_type_1)
SeqIO.write(seq_2, pass_handle_2, out_type_2)
# Write unpaired file 2 records and updated paired key list for finding unpaired file 1 records
if out_args['failed']:
if seq_1 is not None: pass_keys.append(coord_2)
else: SeqIO.write(seq_2, fail_handle_2, out_type_2)
# Print final progress
printProgress(rec_count, seq_count_2, 0.05, start_time)
# Find and write unpaired file 1 records
if out_args['failed']:
start_time = time()
printMessage("Finding unpaired", start_time=start_time)
# Find file 1 unpaired keys
pass_keys = set(pass_keys)
unpaired = set(seq_dict_1).difference(pass_keys)
# Write unpaired file 1 records
for k in unpaired: SeqIO.write(seq_dict_1[k], fail_handle_1, out_type_1)
printMessage("Done", start_time=start_time, end=True)
# Print log
log = OrderedDict()
log['OUTPUT1'] = os.path.basename(pass_handle_1.name)
log['OUTPUT2'] = os.path.basename(pass_handle_2.name)
log['SEQUENCES1'] = seq_count_1
log['SEQUENCES2'] = seq_count_2
log['PASS'] = pair_count
log['END'] = 'PairSeq'
printLog(log)
# Close file handles
pass_handle_1.close()
pass_handle_2.close()
return [(pass_handle_1.name, pass_handle_2.name)]
def renameDbFile(db_file, fields, names, out_args=default_out_args):
"""
Renames fields in a database file
Arguments:
db_file = the database file name
fields = a list of fields to rename
values = a list of new names for fields
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'rename'
log['FILE'] = os.path.basename(db_file)
log['FIELDS'] = ','.join(fields)
log['NAMES'] = ','.join(names)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
# Get header and rename fields
header = (readDbFile(db_file, ig=False)).fieldnames
for f, n in zip(fields, names):
i = header.index(f)
header[i] = n
# Open writer and write new header
# TODO: should modify getDbWriter to take a list of fields
pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab')
pass_writer.writeheader()
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore
# Rename fields
for f, n in zip(fields, names):
rec[n] = rec.pop(f)
# Write
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def insertGaps(db_file, references=None, format=default_format,
out_file=None, out_args=default_out_args):
"""
Inserts IMGT numbering into V fields
Arguments:
db_file : the database file name.
references : folder with germline repertoire files. If None, do not updated alignment columns wtih IMGT gaps.
format : input format.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name
"""
log = OrderedDict()
log['START'] = 'ConvertDb'
log['COMMAND'] = 'imgt'
log['FILE'] = os.path.basename(db_file)
printLog(log)
# Define format operators
try:
reader, writer, schema = getFormatOperators(format)
except ValueError:
printError('Invalid format %s.' % format)
# Open input
db_handle = open(db_file, 'rt')
db_iter = reader(db_handle)
# Check for required columns
try:
required = ['sequence_imgt', 'v_germ_start_imgt']
checkFields(required, db_iter.fields, schema=schema)
except LookupError as e:
printError(e)
# Load references
reference_dict = readGermlines(references)
# Check for IMGT-gaps in germlines
if all('...' not in x for x in reference_dict.values()):
printWarning('Germline reference sequences do not appear to contain IMGT-numbering spacers. Results may be incorrect.')
# Open output writer
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file, out_label='gap', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=schema.out_type)
pass_writer = writer(pass_handle, fields=db_iter.fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Update IMGT fields
imgt_dict = correctIMGTFields(rec, reference_dict)
# Write records
if imgt_dict is not None:
pass_count += 1
rec.setDict(imgt_dict, parse=False)
pass_writer.writeReceptor(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['PASS'] = pass_count
log['FAIL'] = rec_count - pass_count
log['END'] = 'ConvertDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args):
"""
Divides a tab-delimited database file into segments by description tags
Arguments:
db_file = filename of the tab-delimited database file to split
field = the field name by which to split db_file
num_split = the numerical threshold by which to group sequences;
if None treat field as textual
out_args = common output argument dictionary from parseCommonArgs
Returns:
a list of output file names
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'split'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['NUM_SPLIT'] = num_split
printLog(log)
# Open IgRecord reader iter object
reader = readDbFile(db_file, ig=False)
# Determine total numbers of records
rec_count = countDbFile(db_file)
start_time = time()
count = 0
# Sort records into files based on textual field
if num_split is None:
# Create set of unique field tags
tmp_iter = readDbFile(db_file, ig=False)
tag_list = list(set([row[field] for row in tmp_iter]))
# Forbidden characters in filename and replacements
noGood = {'\/':'f','\\':'b','?':'q','\%':'p','*':'s',':':'c',
'\|':'pi','\"':'dq','\'':'sq','<':'gt','>':'lt',' ':'_'}
# Replace forbidden characters in tag_list
tag_dict = {}
for tag in tag_list:
for c,r in noGood.items():
tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \
if c in tag else tag_dict.get(tag, tag))
# Create output handles
handles_dict = {tag:getOutputHandle(db_file,
'%s-%s' % (field, label),
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir'])
for tag, label in tag_dict.items()}
# Create Db writer instances
writers_dict = {tag:getDbWriter(handles_dict[tag], db_file)
for tag in tag_dict}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
# Write row to appropriate file
tag = row[field]
writers_dict[tag].writerow(row)
# Sort records into files based on numeric num_split
else:
num_split = float(num_split)
# Create output handles
handles_dict = {'under':getOutputHandle(db_file,
'under-%.1f' % num_split,
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir']),
'atleast':getOutputHandle(db_file,
'atleast-%.1f' % num_split,
out_type = out_args['out_type'],
out_name = out_args['out_name'],
out_dir = out_args['out_dir'])}
# Create Db writer instances
writers_dict = {'under':getDbWriter(handles_dict['under'], db_file),
'atleast':getDbWriter(handles_dict['atleast'], db_file)}
# Iterate over IgRecords
for row in reader:
printProgress(count, rec_count, 0.05, start_time)
count += 1
tag = row[field]
tag = 'under' if float(tag) < num_split else 'atleast'
writers_dict[tag].writerow(row)
# Write log
printProgress(count, rec_count, 0.05, start_time)
log = OrderedDict()
for i, k in enumerate(handles_dict):
log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
log['RECORDS'] = rec_count
log['PARTS'] = len(handles_dict)
log['END'] = 'ParseDb'
printLog(log)
# Close output file handles
for t in handles_dict: handles_dict[t].close()
return [handles_dict[t].name for t in handles_dict]
def convertToChangeo(db_file, out_file=None, out_args=default_out_args):
"""
Converts an AIRR formatted file into an Change-O formatted file
Arguments:
db_file: the database file name.
out_file : output file name. Automatically generated from the input file if None.
out_args : common output argument dictionary from parseCommonArgs.
Returns:
str : output file name.
"""
log = OrderedDict()
log['START'] = 'ConvertDb'
log['COMMAND'] = 'changeo'
log['FILE'] = os.path.basename(db_file)
printLog(log)
# Open input
db_handle = open(db_file, 'rt')
db_iter = AIRRReader(db_handle)
# Set output fields replacing length with end fields
in_fields = [AIRRSchema.toReceptor(f) for f in db_iter.fields]
out_fields = []
for f in in_fields:
out_fields.append(f)
if f in ReceptorData.end_fields and ReceptorData.end_fields[f][0] in in_fields:
out_fields.append(ReceptorData.end_fields[f][1])
out_fields = list(OrderedDict.fromkeys(out_fields))
out_fields = [ChangeoSchema.fromReceptor(f) for f in out_fields]
# Open output writer
if out_file is not None:
pass_handle = open(out_file, 'w')
else:
pass_handle = getOutputHandle(db_file, out_label='changeo', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type=ChangeoSchema.out_type)
pass_writer = ChangeoWriter(pass_handle, fields=out_fields)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time=start_time)
rec_count += 1
# Write records
pass_writer.writeReceptor(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time=start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['END'] = 'ConvertDb'
printLog(log)
# Close file handles
pass_handle.close()
db_handle.close()
return pass_handle.name
def updateDbFile(db_file, field, values, updates, out_args=default_out_args):
"""
Updates field and value pairs to a database file
Arguments:
db_file = the database file name
field = the field to update
values = a list of values to specifying which rows to update
updates = a list of values to update each value with
out_args = common output argument dictionary from parseCommonArgs
Returns:
the output file name
"""
log = OrderedDict()
log['START'] = 'ParseDb'
log['COMMAND'] = 'update'
log['FILE'] = os.path.basename(db_file)
log['FIELD'] = field
log['VALUES'] = ','.join(values)
log['UPDATES'] = ','.join(updates)
printLog(log)
# Open file handles
db_iter = readDbFile(db_file, ig=False)
pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'],
out_name=out_args['out_name'], out_type='tab')
pass_writer = getDbWriter(pass_handle, db_file)
# Count records
result_count = countDbFile(db_file)
# Iterate over records
start_time = time()
rec_count = pass_count = 0
for rec in db_iter:
# Print progress for previous iteration
printProgress(rec_count, result_count, 0.05, start_time)
rec_count += 1
# Updated values if found
for x, y in zip(values, updates):
if rec[field] == x:
rec[field] = y
pass_count += 1
# Write records
pass_writer.writerow(rec)
# Print counts
printProgress(rec_count, result_count, 0.05, start_time)
log = OrderedDict()
log['OUTPUT'] = os.path.basename(pass_handle.name)
log['RECORDS'] = rec_count
log['UPDATED'] = pass_count
log['END'] = 'ParseDb'
printLog(log)
# Close file handles
pass_handle.close()
return pass_handle.name
def collectEEQueue(alive, result_queue, collect_queue, seq_file, out_args, set_field):
"""
Pulls from results queue, assembles results and manages log and file IO
Arguments:
alive = a multiprocessing.Value boolean controlling whether processing
continues; when False function returns
result_queue = a multiprocessing.Queue holding worker results
collect_queue = a multiprocessing.Queue to store collector return values
seq_file = the sample sequence file name
out_args = common output argument dictionary from parseCommonArgs
set_field = the field defining set membership
Returns:
None
(adds a dictionary of {log: log object, out_files: output file names} to collect_queue)
"""
try:
# Count sets
result_count = countSeqSets(seq_file, set_field, out_args['delimiter'])
# Define empty DataFrames to store assembled results
pos_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float)
qual_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float)
nuc_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float)
set_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float)
# Open log file
if out_args['log_file'] is None:
log_handle = None
else:
log_handle = open(out_args['log_file'], 'w')
except:
alive.value = False
raise
try:
# Iterator over results queue until sentinel object reached
start_time = time()
set_count = seq_count = pass_count = fail_count = 0
while alive.value:
# Get result from queue
if result_queue.empty(): continue
else: result = result_queue.get()
# Exit upon reaching sentinel
if result is None: break
# Print progress for previous iteration
printProgress(set_count, result_count, 0.05, start_time)
# Update counts for iteration
set_count += 1
seq_count += result.data_count
# Sum results
if result:
pass_count += 1
pos_df = pos_df.add(result.results['pos'], fill_value=0)
qual_df = qual_df.add(result.results['qual'], fill_value=0)
nuc_df = nuc_df.add(result.results['nuc'], fill_value=0)
set_df = set_df.add(result.results['set'], fill_value=0)
else:
fail_count += 1
# Write log
printLog(result.log, handle=log_handle)
else:
sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \
% os.getpid())
return None
# Print final progress
printProgress(set_count, result_count, 0.05, start_time)
# Generate log
log = OrderedDict()
for i in range(4):
log['OUTPUT%i' % (i + 1)] = None
log['SETS'] = set_count
log['SEQUENCES'] = seq_count
log['PASS'] = pass_count
log['FAIL'] = fail_count
log['POSITION_ERROR'] = None
log['NUCLEOTIDE_ERROR'] = None
log['QUALITY_ERROR'] = None
log['SET_ERROR'] = None
# Return if no mismatch data
if pass_count == 0:
collect_dict = {'log':log, 'out_files': None}
collect_queue.put(collect_dict)
return None
# Calculate error rates
pos_df['error'] = pos_df['mismatch'] / pos_df['total']
nuc_df['error'] = nuc_df['mismatch'] / nuc_df['total']
qual_df['error'] = qual_df['mismatch'] / qual_df['total']
set_df['error'] = set_df['mismatch'] / set_df['total']
# Convert error to empirical quality score
pos_df['emp_q'] = -10 * np.log10(pos_df['error'])
nuc_df['emp_q'] = -10 * np.log10(nuc_df['error'])
qual_df['emp_q'] = -10 * np.log10(qual_df['error'])
set_df['emp_q'] = -10 * np.log10(set_df['error'])
# Calculate reported quality means
pos_df['rep_q'] = pos_df['q_sum'] / pos_df['total']
nuc_df['rep_q'] = nuc_df['q_sum'] / nuc_df['total']
qual_df['rep_q'] = qual_df['q_sum'] / qual_df['total']
set_df['rep_q'] = set_df['q_sum'] / set_df['total']
# Calculate overall error rate
pos_error = pos_df['mismatch'].sum() / pos_df['total'].sum()
qual_error = qual_df['mismatch'].sum() / qual_df['total'].sum()
nuc_error = nuc_df['mismatch'].sum() / nuc_df['total'].groupby(level='obs').mean().sum()
set_error = set_df['mismatch'].sum() / set_df['total'].sum()
# Build results dictionary
assembled = {'pos':pos_df, 'qual':qual_df, 'nuc':nuc_df, 'set':set_df}
# Write assembled error counts to output files
out_files = writeResults(assembled, seq_file, out_args)
# Update log
for i, f in enumerate(out_files):
log['OUTPUT%i' % (i + 1)] = os.path.basename(f)
log['POSITION_ERROR'] = pos_error
log['NUCLEOTIDE_ERROR'] = nuc_error
log['QUALITY_ERROR'] = qual_error
log['SET_ERROR'] = set_error
# Update collector results
collect_dict = {'log':log, 'out_files': out_files}
collect_queue.put(collect_dict)
except:
alive.value = False
raise
return None
