def writeResults(results, seq_file, out_args): """ Formats results and writes to output files Arguments: results = assembled results dictionary seq_file = the sample sequence file name out_args = common output argument dictionary from parseCommonArgs Returns: a tuple of (position error, nucleotide pairwise error, quality error, sequence set) file names """ pos_df = results['pos'] nuc_df = results['nuc'] qual_df = results['qual'] set_df = results['set'] # Type conversion to int of mismatch and total columns pos_df[['mismatch', 'total']] = pos_df[['mismatch', 'total']].astype(int) nuc_df[['mismatch', 'total']] = nuc_df[['mismatch', 'total']].astype(int) qual_df[['mismatch', 'total']] = qual_df[['mismatch', 'total']].astype(int) set_df[['mismatch', 'total']] = set_df[['mismatch', 'total']].astype(int) # Write to tab delimited files file_args = {'out_dir':out_args['out_dir'], 'out_name':out_args['out_name'], 'out_type':'tab'} with getOutputHandle(seq_file, 'error-position', **file_args) as pos_handle, \ getOutputHandle(seq_file, 'error-quality', **file_args) as qual_handle, \ getOutputHandle(seq_file, 'error-nucleotide', **file_args) as nuc_handle, \ getOutputHandle(seq_file, 'error-set', **file_args) as set_handle: pos_df.to_csv(pos_handle, sep='\t', na_rep='NA', index_label='POSITION', cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f') nuc_df.to_csv(nuc_handle, sep='\t', na_rep='NA', index_label=['OBSERVED', 'REFERENCE'], cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f') qual_df.to_csv(qual_handle, sep='\t', na_rep='NA', index_label='Q', cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f') set_df.to_csv(set_handle, sep='\t', na_rep='NA', index_label='SET_COUNT', cols=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=['REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q'], float_format='%.6f') return (pos_handle.name, qual_handle.name, nuc_handle.name, set_handle.name)
def _open(x, out_file=out_file): if out_file is not None and x == 'pass': handle = open(out_file, 'w') else: handle = getOutputHandle(seq_file, 'convert-%s' % x, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) return handle
def _open(x, in_file, out_name, out_file=out_file): if out_file is not None and x == 'pass': handle = open(out_file, 'w') else: handle = getOutputHandle(in_file, out_label='%s-%s' % (label, x), out_dir=out_args['out_dir'], out_name=out_name, out_type=out_type) return handle
def addDbFile(db_file, fields, values, out_args=default_out_args): """ Adds field and value pairs to a database file Arguments: db_file = the database file name fields = a list of fields to add values = a list of values to assign to all rows of each field out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields) # Count records result_count = countDbFile(db_file) # Define fields and values to append add_dict = {k:v for k,v in zip(fields, values) if k not in db_iter.fieldnames} # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write updated row rec.update(add_dict) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args): """ Adds an index column to a database file Arguments: db_file = the database file name field = the name of the index field to add out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'index' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=field) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Add count and write updated row rec.update({field: rec_count}) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args): """ Deletes entire fields from a database file Arguments: db_file = the database file name fields = a list of fields to drop out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write row pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args): """ Adds an index column to a database file Arguments: db_file = the database file name field = the name of the index field to add out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'index' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=field) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Add count and write updated row rec.update({field:rec_count}) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def writeOffsetFile(primer_file, align_func=runMuscle, align_args={}, reverse=False, out_args=default_out_args): """ Generates an offset table from a sequence file Arguments: primer_file = name of file containing primer sequences align_func = the function to use to align sequence sets align_args = a dictionary of arguments to pass to align_func reverse = if True count tail gaps; if False count head gaps out_args = common output argument dictionary from parseCommonArgs Returns: the name of the offset output file """ log = OrderedDict() log['START'] = 'AlignSets' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(primer_file) log['REVERSE'] = reverse printLog(log) # Read primer file primers = readPrimerFile(primer_file) # Get offset dictionary seq_list = [SeqRecord(Seq(v, IUPAC.ambiguous_dna), id=k) for k, v in primers.items()] offset_dict = getOffsets(seq_list, align_func, align_args, reverse) # Print log and write offsets to file log = OrderedDict() for s in seq_list: log[s.id] = '%s %i' % (s.seq, offset_dict[s.id]) printLog(log) # Write offset table out_tag = 'reverse' if reverse else 'forward' with getOutputHandle(primer_file, 'offsets-%s' % out_tag, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') as out_handle: for k, v in offset_dict.items(): out_handle.write('%s\t%i\n' % (k, v)) # Print final log log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['END'] = 'AlignSets' printLog(log) return out_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args): """ Deletes entire fields from a database file Arguments: db_file = the database file name fields = a list of fields to drop out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write row pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args, cluster_func, cluster_args): """ Assembles results from a queue of individual sequence results and manages log/file I/O Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues if False exit process result_queue = a multiprocessing.Queue holding processQueue results collect_queue = a multiprocessing.Queue to store collector return values db_file = the input database file name out_args = common output argument dictionary from parseCommonArgs cluster_func = the function to call for carrying out clustering on distance matrix cluster_args = a dictionary of arguments to pass to cluster_func Returns: None (adds 'log' and 'out_files' to collect_dict) """ # Open output files try: # Iterate over Ig records to count and order by junction length result_count = 0 records = {} # print 'Reading file...' db_iter = readDbFile(db_file) for rec in db_iter: records[rec.id] = rec result_count += 1 records = OrderedDict(sorted(list(records.items()), key=lambda i: i[1].junction_length)) # Define empty matrix to store assembled results dist_mat = np.zeros((result_count,result_count)) # Count records and define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined successful output handle pass_handle = getOutputHandle(db_file, out_label='clone-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE') # Defined failed cloning output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, out_label='clone-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None fail_writer = None # Open log file if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() row_count = rec_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration if row_count == 0: print('PROGRESS> Assigning clones') printProgress(row_count, result_count, 0.05, start_time) # Update counts for iteration row_count += 1 rec_count += len(result) # Add result row to distance matrix if result: dist_mat[list(range(result_count-len(result),result_count)),result_count-len(result)] = result.results else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Calculate linkage and carry out clustering # print dist_mat clusters = cluster_func(dist_mat, **cluster_args) if dist_mat is not None else None clones = {} # print clusters for i, c in enumerate(clusters): clones.setdefault(c, []).append(records[list(records.keys())[i]]) # Write passed and failed records clone_count = pass_count = fail_count = 0 if clones: for clone in clones.values(): clone_count += 1 for i, rec in enumerate(clone): rec.annotations['CLONE'] = clone_count pass_writer.writerow(rec.toDict()) pass_count += 1 #result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction) else: for i, rec in enumerate(result.data): fail_writer.writerow(rec.toDict()) fail_count += 1 #result.log['CLONE0-%i' % (i + 1)] = str(rec.junction) # Print final progress printProgress(row_count, result_count, 0.05, start_time) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() # Update return list log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) except: alive.value = False raise return None
def writeResults(results, seq_file, out_args): """ Formats results and writes to output files Arguments: results : assembled results dictionary. seq_file : the sample sequence file name. out_args : common output argument dictionary from parseCommonArgs. Returns: tuple: (position error, nucleotide pairwise error, quality error, sequence set) file names """ pos_df = results['pos'] nuc_df = results['nuc'] qual_df = results['qual'] set_df = results['set'] dist_df = results['dist'] thresh_df = results['thresh'] # Type conversion to int of mismatch and total columns pos_df[['mismatch', 'total']] = pos_df[['mismatch', 'total']].astype(int) nuc_df[['mismatch', 'total']] = nuc_df[['mismatch', 'total']].astype(int) qual_df[['mismatch', 'total']] = qual_df[['mismatch', 'total']].astype(int) set_df[['mismatch', 'total']] = set_df[['mismatch', 'total']].astype(int) dist_df[['all']] = dist_df[['all']].astype(int) # Write to tab delimited files file_args = { 'out_dir': out_args['out_dir'], 'out_name': out_args['out_name'], 'out_type': 'tab' } with getOutputHandle(seq_file, 'error-position', **file_args) as pos_handle, \ getOutputHandle(seq_file, 'error-quality', **file_args) as qual_handle, \ getOutputHandle(seq_file, 'error-nucleotide', **file_args) as nuc_handle, \ getOutputHandle(seq_file, 'error-set', **file_args) as set_handle, \ getOutputHandle(seq_file, 'distance-set', **file_args) as dist_handle, \ getOutputHandle(seq_file, 'threshold-set', **file_args) as thresh_handle: pos_df.to_csv(pos_handle, sep='\t', na_rep='NA', index=True, index_label='POSITION', columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=[ 'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q' ], float_format='%.6f') nuc_df.to_csv(nuc_handle, sep='\t', na_rep='NA', index=True, index_label=['REFERENCE', 'OBSERVED'], columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=[ 'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q' ], float_format='%.6f') qual_df.to_csv( qual_handle, sep='\t', na_rep='NA', index=True, index_label='Q', columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=[ 'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q' ], float_format='%.6f') set_df.to_csv(set_handle, sep='\t', na_rep='NA', index=True, index_label='SET_COUNT', columns=['rep_q', 'mismatch', 'total', 'error', 'emp_q'], header=[ 'REPORTED_Q', 'MISMATCHES', 'OBSERVATIONS', 'ERROR', 'EMPIRICAL_Q' ], float_format='%.6f') dist_df.to_csv(dist_handle, sep='\t', na_rep='NA', index=True, index_label='DISTANCE', columns=['all'], header=['ALL'], float_format='%.6f') thresh_df.to_csv(thresh_handle, sep='\t', na_rep='NA', index=True, index_label='TYPE', columns=['thresh'], header=['THRESHOLD'], float_format='%.6f') return (pos_handle.name, qual_handle.name, nuc_handle.name, set_handle.name, dist_handle.name, thresh_handle.name)
def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sequences meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles out_type = 'fasta' db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Get SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter']) # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, out_type) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def assembleCloneGermline(db_file, repo, seq_field=default_seq_field, v_field=default_v_field, germ_types=default_germ_types, out_args=default_out_args): """ Assemble one germline sequence for each clone in a tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'True' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader references = readRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS'] add_fields += ['GERMLINE_V_CALL'] add_fields += ['GERMLINE_D_CALL'] add_fields += ['GERMLINE_J_CALL'] # Create output file handle and Db writer writers = {} pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None writers['fail'] = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) counts = {} clone_count = counts['pass'] = counts['fail'] = 0 # Iterate over rows clone = 'initial' clone_dict = OrderedDict() for i, row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) # Clone isn't over yet if row.get('CLONE', '') == clone: clone_dict[i] = row # Clone just finished elif clone_dict: clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, references, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Now deal with current row (first of next clone) clone = row['CLONE'] clone_dict = OrderedDict([(i, row)]) # Last case is only for first row of file else: clone = row['CLONE'] clone_dict = OrderedDict([(i, row)]) clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, references, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Print log printProgress(i + 1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = counts['pass'] log['FAIL'] = counts['fail'] log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def sortDbFile(db_file, field, numeric=False, descend=False, out_args=default_out_args): """ Sorts records by values in an annotation field Arguments: db_file = the database filename field = the field name to sort by numeric = if True sort field numerically; if False sort field alphabetically descend = if True sort in descending order; if False sort in ascending order out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUMERIC'] = numeric printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Store all records in a dictionary start_time = time() printMessage("Indexing: Running", start_time=start_time) db_dict = {i:r for i, r in enumerate(db_iter)} result_count = len(db_dict) # Sort db_dict by field values tag_dict = {k:v[field] for k, v in db_dict.items()} if numeric: tag_dict = {k:float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend) printMessage("Indexing: Done", start_time=start_time, end=True) # Iterate over records start_time = time() rec_count = 0 for key in sorted_keys: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write records pass_writer.writerow(db_dict[key]) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def collectDbQueue(alive, result_queue, collect_queue, db_file, task_label, out_args, add_fields=None): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive : multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue : multiprocessing.Queue holding worker results collect_queue : multiprocessing.Queue to store collector return values db_file : Database file name task_label : Task label used to tag the output files out_args : Common output argument dictionary from parseCommonArgs add_fields : List of fields added to the writer not present in the in_file; if None do not add fields Returns: None : Adds a dictionary with key value pairs to collect_queue containing 'log' defining a log object, 'out_files' defining the output file names """ try: result_count = countDbFile(db_file) # Define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined valid alignment output handle pass_handle = getOutputHandle(db_file, '%s-pass' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) # Defined failed alignment output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, '%s-fail' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() set_count = rec_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(pass_count, result_count, 0.05, start_time) # Update counts for current iteration set_count += 1 rec_count += result.data_count # Write log printLog(result.log, handle=log_handle) # Write alignments if result: pass_count += result.data_count for rec in result.results: pass_writer.writerow(rec.toDict()) else: fail_count += result.data_count if fail_handle is not None: for rec in result.data: pass_writer.writerow(rec.toDict()) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(pass_count, result_count, 0.05, start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GROUPS'] = set_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() except: alive.value = False raise return None
def collectDbQueue(alive, result_queue, collect_queue, db_file, task_label, out_args, add_fields=None): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive : multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue : multiprocessing.Queue holding worker results collect_queue : multiprocessing.Queue to store collector return values db_file : Database file name task_label : Task label used to tag the output files out_args : Common output argument dictionary from parseCommonArgs add_fields : List of fields added to the writer not present in the in_file; if None do not add fields Returns: None : Adds a dictionary with key value pairs to collect_queue containing 'log' defining a log object, 'out_files' defining the output file names """ try: result_count = countDbFile(db_file) # Define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined valid alignment output handle pass_handle = getOutputHandle(db_file, '%s-pass' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) # Defined failed alignment output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, '%s-fail' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() set_count = rec_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(pass_count, result_count, 0.05, start_time) # Update counts for current iteration set_count += 1 rec_count += result.data_count # Write log if result.log is not None: printLog(result.log, handle=log_handle) # Write alignments if result: pass_count += result.data_count if isinstance(result.results, IgRecord): pass_writer.writerow(result.results.toDict()) else: for rec in result.results: pass_writer.writerow(rec.toDict()) else: fail_count += result.data_count if fail_handle is not None: if isinstance(result.data, IgRecord): pass_writer.writerow(result.data.toDict()) else: for rec in result.data: fail_writer.writerow(rec.toDict()) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(pass_count, result_count, 0.05, start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GROUPS'] = set_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log': log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() except: alive.value = False raise return None
def writeOffsetFile(primer_file, align_func=runMuscle, align_args={}, reverse=False, out_file=None, out_args=default_out_args): """ Generates an offset table from a sequence file Arguments: primer_file : name of file containing primer sequences. align_func : the function to use to align sequence sets. align_args : a dictionary of arguments to pass to align_func. reverse : if True count tail gaps; if False count head gaps. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: the name of the offset output file. """ log = OrderedDict() log['START'] = 'AlignSets' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(primer_file) log['REVERSE'] = reverse printLog(log) # Read primer file primers = readPrimerFile(primer_file) # Get offset dictionary seq_list = [SeqRecord(Seq(v, IUPAC.ambiguous_dna), id=k) for k, v in primers.items()] offset_dict = getOffsets(seq_list, align_func, align_args, reverse) # Print log and write offsets to file log = OrderedDict() for s in seq_list: log[s.id] = '%s %i' % (s.seq, offset_dict[s.id]) printLog(log) # Open output handle if out_file is not None: out_handle = open(out_file, 'w') else: out_tag = 'reverse' if reverse else 'forward' out_handle = getOutputHandle(primer_file, 'offsets-%s' % out_tag, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Write offset table for k, v in offset_dict.items(): out_handle.write('%s\t%i\n' % (k, v)) # Print final log log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['END'] = 'AlignSets' printLog(log) # Close handle out_handle.close() return out_handle.name
def collapseSeq(seq_file, max_missing=default_max_missing, uniq_fields=None, copy_fields=None, copy_actions=None, max_field=None, min_field=None, inner=False, keep_missing=False, out_args=default_out_args): """ Removes duplicate sequences from a file Arguments: seq_file = filename of the sequence file to sample from max_missing = number of ambiguous characters to allow in a unique sequence uniq_fields = a list of annotations that define a sequence as unique if they differ copy_fields = a list of annotations to copy into unique sequence annotations copy_actions = the list of collapseAnnotation actions to take on copy_fields max_field = a numeric field whose maximum value determines the retained sequence min_field = a numeric field whose minimum value determines the retained sequence inner = if True exclude consecutive outer ambiguous characters from iterations and matching keep_missing = if True retain sequences with more ambiguous characters than max_missing as unique out_args = common output argument dictionary from parseCommonArgs Returns: the collapsed output file name """ log = OrderedDict() log['START'] = 'CollapseSeq' log['FILE'] = os.path.basename(seq_file) log['MAX_MISSING'] = max_missing log['UNIQ_FIELDS'] = ','.join([str(x) for x in uniq_fields]) \ if uniq_fields is not None else None log['COPY_FIELDS'] = ','.join([str(x) for x in copy_fields]) \ if copy_fields is not None else None log['COPY_ACTIONS'] = ','.join([str(x) for x in copy_actions]) \ if copy_actions is not None else None log['MAX_FIELD'] = max_field log['MIN_FIELD'] = min_field log['INNER'] = inner log['KEEP_MISSING'] = keep_missing printLog(log) # TODO: storing all sequences in memory is faster # Read input file in_type = getFileType(seq_file) #seq_dict = readSeqFile(seq_file, index=True) seq_dict = SeqIO.to_dict(readSeqFile(seq_file, index=False)) if out_args['out_type'] is None: out_args['out_type'] = in_type # Count total sequences rec_count = len(seq_dict) # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') # Find sequences with duplicates uniq_dict = {} # Added list typing for compatibility issue with Python 2.7.5 on OS X # TypeError: object of type 'dictionary-keyiterator' has no len() search_keys = list(seq_dict.keys()) dup_keys = [] for n in range(0, max_missing + 1): # Find unique sequences uniq_dict, search_keys, dup_list = findUniqueSeq(uniq_dict, search_keys, seq_dict, n, uniq_fields, copy_fields, max_field, min_field, inner, out_args['delimiter']) # Update list of duplicates dup_keys.extend(dup_list) # Update log log = OrderedDict() log['ITERATION'] = n + 1 log['MISSING'] = n log['UNIQUE'] = len(uniq_dict) log['DUPLICATE'] = len(dup_keys) log['UNDETERMINED'] = len(search_keys) printLog(log, handle=log_handle) # Break if no keys to search remain if len(search_keys) == 0: break # Write unique sequences with getOutputHandle(seq_file, 'collapse-unique', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) \ as uniq_handle: for val in uniq_dict.values(): # Define output sequence out_seq = val[0] out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter']) out_app = OrderedDict() if copy_fields is not None and copy_actions is not None: for f, a, s in zip(copy_fields, copy_actions, val[3:]): out_app[f] = s out_app = collapseAnnotation(out_app, a, f, delimiter=out_args['delimiter']) out_ann.pop(f, None) out_app['DUPCOUNT'] = val[1] out_ann = mergeAnnotation(out_ann, out_app, delimiter=out_args['delimiter']) out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter']) out_seq.description = '' # Write unique sequence SeqIO.write(out_seq, uniq_handle, out_args['out_type']) # Write sequence with high missing character counts if keep_missing: for k in search_keys: out_seq = seq_dict[k] out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter']) out_ann = mergeAnnotation(out_ann, {'DUPCOUNT':1}, delimiter=out_args['delimiter']) out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter']) out_seq.description = '' SeqIO.write(out_seq, uniq_handle, out_args['out_type']) # Write sequence with high missing character counts if out_args['failed'] and not keep_missing: with getOutputHandle(seq_file, 'collapse-undetermined', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) \ as missing_handle: for k in search_keys: SeqIO.write(seq_dict[k], missing_handle, out_args['out_type']) if out_args['failed']: # Write duplicate sequences with getOutputHandle(seq_file, 'collapse-duplicate', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) \ as dup_handle: for k in dup_keys: SeqIO.write(seq_dict[k], dup_handle, out_args['out_type']) # Print log log = OrderedDict() log['OUTPUT'] = os.path.basename(uniq_handle.name) log['SEQUENCES'] = rec_count log['UNIQUE'] = len(uniq_dict) log['DUPLICATE'] = len(dup_keys) log['UNDETERMINED'] = len(search_keys) log['END'] = 'CollapseSeq' printLog(log) # Close file handles if log_handle is not None: log_handle.close() return uniq_handle.name
def assembleCloneGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Assemble one germline sequence for each clone in a tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'True' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader repo_dict = getRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] # Create output file handle and Db writer writers = {} pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None writers['fail'] = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) counts = {} clone_count = counts['pass'] = counts['fail'] = 0 # Iterate over rows clone = 'initial' clone_dict = OrderedDict() for i,row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) # Clone isn't over yet if row.get('CLONE','') == clone: clone_dict[row["SEQUENCE_ID"]] = row # Clone just finished elif clone_dict: clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Now deal with current row (first of next clone) clone = row['CLONE'] clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)]) # Last case is only for first row of file else: clone = row['CLONE'] clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)]) clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Print log printProgress(i+1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = counts['pass'] log['FAIL'] = counts['fail'] log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Write germline sequences to tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'False' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader repo_dict = getRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] # Create output file handle and Db writer pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None fail_writer = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) pass_count = fail_count = 0 # Iterate over rows for i,row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) result_log, germlines = joinGermline(row, repo_dict, germ_types, v_field, seq_field) # Add germline field(s) to dictionary if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full'] if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask'] if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly'] # Write row to pass or fail file if 'ERROR' in result_log: fail_count += 1 if fail_writer is not None: fail_writer.writerow(row) else: result_log['SEQUENCE'] = row[seq_field] result_log['GERMLINE'] = germlines['full'] result_log['REGIONS'] = germlines['regions'] pass_count += 1 pass_writer.writerow(row) printLog(result_log, handle=log_handle) # Print log printProgress(i+1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def estimateBarcode(seq_file, barcode_field=default_barcode_field, distance_types=default_distance_types, out_args=default_out_args): """ Calculates error rates of barcode sequences Arguments: seq_file : the sample sequence file name barcode_field : the annotation field containing barcode sequences. distance_types : distance types to include. out_args : common output argument dictionary from parseCommonArgs Returns: tuple: names of the output files. """ # Function to extract to make SeqRecord object from a barcode annotation def _barcode(seq, field=barcode_field, delimiter=out_args['delimiter']): header = parseAnnotation(seq.description, delimiter=delimiter) return header[field] # Print parameter info log = OrderedDict() log['START'] = 'EstimateError' log['COMMAND'] = 'barcode' log['FILE'] = os.path.basename(seq_file) log['BARCODE_FIELD'] = barcode_field printLog(log) # Count sequence file and parse into a list of SeqRecords result_count = countSeqFile(seq_file) barcode_iter = (_barcode(x) for x in readSeqFile(seq_file)) # Compute bin_count defaults to the length of the barcode + 1 bin_count = len(_barcode(next(readSeqFile(seq_file)))) + 1 mismatch = initializeMismatchDictionary(0, distance_types=distance_types, bin_count=bin_count) # Calculate distances distance_mismatch = calculateDistances(barcode_iter, bin_count=bin_count) mismatch['dist'] = { header: distance_mismatch[header] for header in distance_types } # Generate a df dist_df = pd.DataFrame.from_dict(mismatch['dist']) dist_df.index = dist_df.index / len(dist_df.index) dist_df[['all']] = dist_df[['all']].astype(int) #find the threshold (average minimum between 0 and 0.75) dist = mismatch['dist']['all'] thresh_df = pd.DataFrame.from_dict({'thresh': {'ALL': dist_df.index[int(np.mean([index for index in np.argsort(dist[:int(len(dist)*0.75)]) \ if dist[index] == np.min(dist)]))]} }) file_args = { 'out_dir': out_args['out_dir'], 'out_name': out_args['out_name'], 'out_type': 'tab' } # Output as tsv with getOutputHandle(seq_file, 'distance-barcode', **file_args) as dist_handle, \ getOutputHandle(seq_file, 'threshold-barcode', **file_args) as thresh_handle: dist_df.to_csv(dist_handle, sep='\t', na_rep='NA', index=True, index_label='DISTANCE', columns=['all'], header=['ALL'], float_format='%.6f') thresh_df.to_csv(thresh_handle, sep='\t', na_rep='NA', index=True, index_label='TYPE', columns=['thresh'], header=['THRESHOLD'], float_format='%.6f') # Update log log['OUTPUT1'] = os.path.basename(dist_handle.name) log['OUTPUT2'] = os.path.basename(thresh_handle.name) log['SEQUENCES'] = result_count log['ALL_THRESHOLD'] = '%.6f' % thresh_df['thresh']['ALL'] log['END'] = 'EstimateError' printLog(log) return (dist_handle.name, thresh_handle.name)
def convertDbClip(db_file, id_field=default_id_field, seq_field=default_seq_field, germ_field=default_germ_field, cluster_field=None, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sample sequences germ_field = the field containing germline sequences cluster_field = the field containing clonal groupings if None write the germline for each record meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field log['GERM_FIELD'] = germ_field log['CLUSTER_FIELD'] = cluster_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='clip') # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = germ_count = pass_count = fail_count = 0 cluster_last = None for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Update cluster ID cluster = rec.get(cluster_field, None) # Get germline SeqRecord when needed if cluster_field is None: germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields, delimiter=out_args['delimiter']) germ.id = '>' + germ.id elif cluster != cluster_last: germ = getDbSeqRecord(rec, cluster_field, germ_field, delimiter=out_args['delimiter']) germ.id = '>' + germ.id else: germ = None # Get read SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, delimiter=out_args['delimiter']) # Write germline if germ is not None: germ_count += 1 SeqIO.write(germ, pass_handle, 'fasta') # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, 'fasta') else: fail_count += 1 # Set last cluster ID cluster_last = cluster # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GERMLINES'] = germ_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def collectPairQueue(alive, result_queue, collect_queue, result_count, seq_file_1, seq_file_2, out_args): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue = a multiprocessing.Queue holding worker results collect_queue = a multiprocessing.Queue holding collector return values result_count = the number of expected assembled sequences seq_file_1 = the first sequence file name seq_file_2 = the second sequence file name out_args = common output argument dictionary from parseCommonArgs Returns: None (adds a dictionary of {log: log object, out_files: output file names} to collect_queue) """ try: # Count records and define output format out_type = getFileType(seq_file_1) if out_args['out_type'] is None \ else out_args['out_type'] # Defined valid assembly output handle pass_handle = getOutputHandle(seq_file_1, 'assemble-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Defined failed assembly output handles if out_args['failed']: # Define output name if out_args['out_name'] is None: out_name_1 = out_name_2 = None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] fail_handle_1 = getOutputHandle(seq_file_1, 'assemble-fail', out_dir=out_args['out_dir'], out_name=out_name_1, out_type=out_type) fail_handle_2 = getOutputHandle(seq_file_2, 'assemble-fail', out_dir=out_args['out_dir'], out_name=out_name_2, out_type=out_type) else: fail_handle_1 = None fail_handle_2 = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() iter_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(iter_count, result_count, 0.05, start_time) # Update counts for iteration iter_count += 1 # Write log printLog(result.log, handle=log_handle) # Write assembled sequences if result: pass_count += 1 SeqIO.write(result.results, pass_handle, out_type) else: fail_count += 1 if fail_handle_1 is not None and fail_handle_2 is not None: SeqIO.write(result.data[0], fail_handle_1, out_type) SeqIO.write(result.data[1], fail_handle_2, out_type) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(iter_count, result_count, 0.05, start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['PAIRS'] = iter_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) # Close file handles pass_handle.close() if fail_handle_1 is not None: fail_handle_1.close() if fail_handle_2 is not None: fail_handle_2.close() if log_handle is not None: log_handle.close() except: alive.value = False raise return None
def tableHeaders(seq_file, fields, out_file=None, out_args=default_out_args): """ Builds a table of sequence header annotations Arguments: seq_file : the sequence file name. fields : the list of fields to output. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: output table file name """ log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(seq_file) printLog(log) # Open file handles seq_iter = readSeqFile(seq_file) if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(seq_file, 'headers', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Count records result_count = countSeqFile(seq_file) # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time=start_time) # Get annotations seq_count += 1 ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter']) # Write records if ann: pass_count += 1 out_writer.writerow(ann) else: fail_count += 1 # Print counts printProgress(seq_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def selectDbFile(db_file, fields, values, logic='any', regex=False, out_args=default_out_args): """ Selects records from a database file Arguments: db_file = the database file name fields = a list of fields to check for selection criteria values = a list of values defining selection targets logic = one of 'any' or 'all' defining whether one or all fields must have a match. regex = if False do exact full string matches; if True allow partial regex matches. out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define string match function if regex: def _match_func(x, patterns): return any([re.search(p, x) for p in patterns]) else: def _match_func(x, patterns): return x in patterns # Define logic function if logic == 'any': _logic_func = any elif logic == 'all': _logic_func = all # Print console log log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'select' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) log['REGEX'] =regex printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Check for selection values in all fields select = _logic_func([_match_func(rec.get(f, False), values) for f in fields]) # Write sequences if select: pass_count += 1 pass_writer.writerow(rec) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['SELECTED'] = pass_count log['DISCARDED'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def modifyHeaders(seq_file, modify_func, modify_args, out_file=None, out_args=default_out_args): """ Modifies sequence headers Arguments: seq_file : the sequence file name. modify_func : the function defining the modification operation. modify_args : a dictionary of arguments to pass to modify_func. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: output file name. """ # Define subcommand label dictionary cmd_dict = {addHeader: 'add', copyHeader: 'copy', collapseHeader: 'collapse', deleteHeader: 'delete', expandHeader: 'expand', renameHeader: 'rename'} # Print parameter info log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__) log['FILE'] = os.path.basename(seq_file) for k in sorted(modify_args): v = modify_args[k] log[k.upper()] = ','.join(v) if isinstance(v, list) else v printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Count records result_count = countSeqFile(seq_file) # Iterate over sequences start_time = time() seq_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time=start_time) #Update counts seq_count += 1 # Modify header header = parseAnnotation(seq.description, delimiter=out_args['delimiter']) header = modify_func(header, delimiter=out_args['delimiter'], **modify_args) # Write new sequence seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter']) seq.description = '' SeqIO.write(seq, out_handle, out_args['out_type']) # print counts printProgress(seq_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def clusterBarcodes(seq_file, ident=default_cluster_ident, length_ratio=default_length_ratio, barcode_field=default_barcode_field, cluster_field=default_cluster_field, cluster_prefix=default_cluster_prefix, cluster_tool=default_cluster_tool, cluster_exec=default_cluster_exec, out_file=None, out_args=default_out_args, nproc=None): """ Performs clustering on sets of sequences Arguments: seq_file : the sample sequence file name. ident : the identity threshold for clustering sequences. length_ratio : minimum short/long length ratio allowed within a cluster. barcode_field : the annotation field containing barcode sequences. cluster_field : the name of the output cluster field. cluster_prefix : string defining a prefix for the cluster identifier. seq_start : the start position to trim sequences at before clustering. seq_end : the end position to trim sequences at before clustering. cluster_tool : the clustering tool to use; one of cd-hit or usearch. cluster_exec : the path to the executable for usearch. out_file : output file name. Automatically generated from the input file if None. out_args : output arguments. nproc : the number of processQueue processes; if None defaults to the number of CPUs. Returns: str: the clustered output file name """ # Function to modify SeqRecord header with cluster identifier def _header(seq, cluster, field=cluster_field, prefix=cluster_prefix, delimiter=out_args['delimiter']): label = '%s%i' % (prefix, cluster) header = parseAnnotation(seq.description, delimiter=delimiter) header = mergeAnnotation(header, {field: label}, delimiter=delimiter) seq.id = seq.name = flattenAnnotation(header, delimiter=delimiter) seq.description = '' return seq # Function to extract to make SeqRecord object from a barcode annotation def _barcode(seq, field=barcode_field, delimiter=out_args['delimiter']): header = parseAnnotation(seq.description, delimiter=delimiter) return SeqRecord(Seq(header[field]), id=seq.id) # Print parameter info log = OrderedDict() log['START'] = 'ClusterSets' log['COMMAND'] = 'barcode' log['FILE'] = os.path.basename(seq_file) log['IDENTITY'] = ident log['BARCODE_FIELD'] = barcode_field log['CLUSTER_FIELD'] = cluster_field log['CLUSTER_PREFIX'] = cluster_prefix log['CLUSTER_TOOL'] = cluster_tool log['NPROC'] = nproc printLog(log) # Set cluster tool try: cluster_func = map_cluster_tool.get(cluster_tool) except: printError('Invalid clustering tool %s.' % cluster_tool) # Check the minimum identity if ident < min_cluster_ident[cluster_tool]: printError('Minimum identity %s too low for clustering tool %s.' % (str(ident), cluster_tool)) # Count sequence file and parse into a list of SeqRecords result_count = countSeqFile(seq_file) barcode_iter = (_barcode(x) for x in readSeqFile(seq_file)) # Perform clustering start_time = time() printMessage('Running %s' % cluster_tool, start_time=start_time, width=25) cluster_dict = cluster_func(barcode_iter, ident=ident, length_ratio=length_ratio, seq_start=0, seq_end=None, threads=nproc, cluster_exec=cluster_exec) printMessage('Done', start_time=start_time, end=True, width=25) # Determine file type if out_args['out_type'] is None: out_args['out_type'] = getFileType(seq_file) # Open output file handles if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(seq_file, 'cluster-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Open indexed sequence file seq_dict = readSeqFile(seq_file, index=True) # Iterate over sequence records and update header with cluster annotation start_time = time() rec_count = pass_count = 0 for cluster, id_list in cluster_dict.items(): printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += len(id_list) # TODO: make a generator. Figure out how to get pass_count updated # Define output sequences seq_output = [_header(seq_dict[x], cluster) for x in id_list] # Write output pass_count += len(seq_output) SeqIO.write(seq_output, pass_handle, out_args['out_type']) # Update progress printProgress(rec_count, result_count, 0.05, start_time=start_time) # Print log log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLUSTERS'] = len(cluster_dict) log['SEQUENCES'] = result_count log['PASS'] = pass_count log['FAIL'] = rec_count - pass_count log['END'] = 'ClusterSets' printLog(log) # Close handles pass_handle.close() return pass_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args): """ Renames fields in a database file Arguments: db_file = the database file name fields = a list of fields to rename values = a list of new names for fields out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'rename' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['NAMES'] = ','.join(names) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Get header and rename fields header = (readDbFile(db_file, ig=False)).fieldnames for f, n in zip(fields, names): i = header.index(f) header[i] = n # Open writer and write new header # TODO: should modify getDbWriter to take a list of fields pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab') pass_writer.writeheader() # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore # Rename fields for f, n in zip(fields, names): rec[n] = rec.pop(f) # Write pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def convertDbBaseline(db_file, id_field=default_id_field, seq_field=default_seq_field, germ_field=default_germ_field, cluster_field=None, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sample sequences germ_field = the field containing germline sequences cluster_field = the field containing clonal groupings if None write the germline for each record meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field log['GERM_FIELD'] = germ_field log['CLUSTER_FIELD'] = cluster_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='clip') # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = germ_count = pass_count = fail_count = 0 cluster_last = None for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Update cluster ID cluster = rec.get(cluster_field, None) # Get germline SeqRecord when needed if cluster_field is None: germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields, delimiter=out_args['delimiter']) germ.id = '>' + germ.id elif cluster != cluster_last: germ = getDbSeqRecord(rec, cluster_field, germ_field, delimiter=out_args['delimiter']) germ.id = '>' + germ.id else: germ = None # Get read SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, delimiter=out_args['delimiter']) # Write germline if germ is not None: germ_count += 1 SeqIO.write(germ, pass_handle, 'fasta') # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, 'fasta') else: fail_count += 1 # Set last cluster ID cluster_last = cluster # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GERMLINES'] = germ_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args): """ Divides a tab-delimited database file into segments by description tags Arguments: db_file = filename of the tab-delimited database file to split field = the field name by which to split db_file num_split = the numerical threshold by which to group sequences; if None treat field as textual out_args = common output argument dictionary from parseCommonArgs Returns: a list of output file names """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'split' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUM_SPLIT'] = num_split printLog(log) # Open IgRecord reader iter object reader = readDbFile(db_file, ig=False) # Determine total numbers of records rec_count = countDbFile(db_file) start_time = time() count = 0 # Sort records into files based on textual field if num_split is None: # Create set of unique field tags tmp_iter = readDbFile(db_file, ig=False) tag_list = list(set([row[field] for row in tmp_iter])) # Forbidden characters in filename and replacements noGood = { '\/': 'f', '\\': 'b', '?': 'q', '\%': 'p', '*': 's', ':': 'c', '\|': 'pi', '\"': 'dq', '\'': 'sq', '<': 'gt', '>': 'lt', ' ': '_' } # Replace forbidden characters in tag_list tag_dict = {} for tag in tag_list: for c, r in noGood.items(): tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \ if c in tag else tag_dict.get(tag, tag)) # Create output handles handles_dict = { tag: getOutputHandle(db_file, '%s-%s' % (field, label), out_type=out_args['out_type'], out_name=out_args['out_name'], out_dir=out_args['out_dir']) for tag, label in tag_dict.items() } # Create Db writer instances writers_dict = { tag: getDbWriter(handles_dict[tag], db_file) for tag in tag_dict } # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 # Write row to appropriate file tag = row[field] writers_dict[tag].writerow(row) # Sort records into files based on numeric num_split else: num_split = float(num_split) # Create output handles handles_dict = { 'under': getOutputHandle(db_file, 'under-%.1f' % num_split, out_type=out_args['out_type'], out_name=out_args['out_name'], out_dir=out_args['out_dir']), 'atleast': getOutputHandle(db_file, 'atleast-%.1f' % num_split, out_type=out_args['out_type'], out_name=out_args['out_name'], out_dir=out_args['out_dir']) } # Create Db writer instances writers_dict = { 'under': getDbWriter(handles_dict['under'], db_file), 'atleast': getDbWriter(handles_dict['atleast'], db_file) } # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 tag = row[field] tag = 'under' if float(tag) < num_split else 'atleast' writers_dict[tag].writerow(row) # Write log printProgress(count, rec_count, 0.05, start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['RECORDS'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'ParseDb' printLog(log) # Close output file handles for t in handles_dict: handles_dict[t].close() return [handles_dict[t].name for t in handles_dict]
def addDbFile(db_file, fields, values, out_args=default_out_args): """ Adds field and value pairs to a database file Arguments: db_file = the database file name fields = a list of fields to add values = a list of values to assign to all rows of each field out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields) # Count records result_count = countDbFile(db_file) # Define fields and values to append add_dict = { k: v for k, v in zip(fields, values) if k not in db_iter.fieldnames } # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write updated row rec.update(add_dict) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def updateDbFile(db_file, field, values, updates, out_args=default_out_args): """ Updates field and value pairs to a database file Arguments: db_file = the database file name field = the field to update values = a list of values to specifying which rows to update updates = a list of values to update each value with out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'update' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['VALUES'] = ','.join(values) log['UPDATES'] = ','.join(updates) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Updated values if found for x, y in zip(values, updates): if rec[field] == x: rec[field] = y pass_count += 1 # Write records pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['UPDATED'] = pass_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def tableLog(record_file, fields, out_args=default_out_args): """ Converts a pRESTO log to a table of annotations Arguments: record_file = the log file name fields = the list of fields to output out_args = common output argument dictionary from parseCommonArgs Returns: the output table file name """ log = OrderedDict() log['START'] = 'ParseLog' log['FILE'] = os.path.basename(record_file) printLog(log) # Open file handles log_handle = open(record_file, 'rU') out_handle = getOutputHandle(record_file, 'table', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over log records start_time = time() record = '' rec_count = pass_count = fail_count = 0 for line in log_handle: if line.strip() == '' and record: # Print progress for previous iteration printProgress(rec_count, None, 1e5, start_time) # Parse record block rec_count += 1 record_dict = parseLogRecord(record) # Write records if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Empty record string record = '' else: # Append to record record += line else: # Write final record if record: record_dict = parseLogRecord(record) if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Print counts printProgress(rec_count, None, 1e5, start_time, end=True) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseLog' printLog(log) # Close file handles log_handle.close() out_handle.close() return log_handle.name
def collapseSeq(seq_file, max_missing=default_max_missing, uniq_fields=None, copy_fields=None, copy_actions=None, max_field=None, min_field=None, inner=False, keep_missing=False, out_file=None, out_args=default_out_args): """ Removes duplicate sequences from a file Arguments: seq_file : filename of the sequence file to sample from. max_missing : number of ambiguous characters to allow in a unique sequence. uniq_fields : a list of annotations that define a sequence as unique if they differ. copy_fields : a list of annotations to copy into unique sequence annotations. copy_actions : the list of collapseAnnotation actions to take on copy_fields. max_field : a numeric field whose maximum value determines the retained sequence. min_field : a numeric field whose minimum value determines the retained sequence. inner : if True exclude consecutive outer ambiguous characters from iterations and matching. keep_missing : if True retain sequences with more ambiguous characters than max_missing as unique. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: the collapsed output file name. """ log = OrderedDict() log['START'] = 'CollapseSeq' log['FILE'] = os.path.basename(seq_file) log['MAX_MISSING'] = max_missing log['UNIQ_FIELDS'] = ','.join([str(x) for x in uniq_fields]) \ if uniq_fields is not None else None log['COPY_FIELDS'] = ','.join([str(x) for x in copy_fields]) \ if copy_fields is not None else None log['COPY_ACTIONS'] = ','.join([str(x) for x in copy_actions]) \ if copy_actions is not None else None log['MAX_FIELD'] = max_field log['MIN_FIELD'] = min_field log['INNER'] = inner log['KEEP_MISSING'] = keep_missing printLog(log) # Read input file in_type = getFileType(seq_file) seq_dict = SeqIO.to_dict(readSeqFile(seq_file, index=False)) if out_args['out_type'] is None: out_args['out_type'] = in_type # Count total sequences rec_count = len(seq_dict) # Open unique record output handle if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(seq_file, 'collapse-unique', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') # Find sequences with duplicates uniq_dict = {} # Added list typing for compatibility issue with Python 2.7.5 on OS X # TypeError: object of type 'dictionary-keyiterator' has no len() search_keys = list(seq_dict.keys()) dup_keys = [] for n in range(0, max_missing + 1): # Find unique sequences uniq_dict, search_keys, dup_list = findUniqueSeq(uniq_dict, search_keys, seq_dict, n, uniq_fields, copy_fields, max_field, min_field, inner, out_args['delimiter']) # Update list of duplicates dup_keys.extend(dup_list) # Break if no keys to search remain if len(search_keys) == 0: break # Write unique sequences for val in uniq_dict.values(): # Define output sequence out_seq = val.seq out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter']) out_app = OrderedDict() if copy_fields is not None and copy_actions is not None: for f, a in zip(copy_fields, copy_actions): x = collapseAnnotation(val.annotations, a, f, delimiter=out_args['delimiter']) out_app[f] = x[f] out_ann.pop(f, None) out_app['DUPCOUNT'] = val.count out_ann = mergeAnnotation(out_ann, out_app, delimiter=out_args['delimiter']) out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter']) out_seq.description = '' # Write unique sequence SeqIO.write(out_seq, pass_handle, out_args['out_type']) # Update log log = OrderedDict() log['HEADER'] = out_seq.id log['DUPCOUNT'] = val.count for i, k in enumerate(val.keys, start=1): log['ID%i' % i] = k for i, k in enumerate(val.keys, start=1): log['SEQ%i' % i] = str(seq_dict[k].seq) printLog(log, handle=log_handle) # Write sequence with high missing character counts if keep_missing: for k in search_keys: out_seq = seq_dict[k] out_ann = parseAnnotation(out_seq.description, delimiter=out_args['delimiter']) out_ann = mergeAnnotation(out_ann, {'DUPCOUNT':1}, delimiter=out_args['delimiter']) out_seq.id = out_seq.name = flattenAnnotation(out_ann, delimiter=out_args['delimiter']) out_seq.description = '' SeqIO.write(out_seq, pass_handle, out_args['out_type']) # Write sequence with high missing character counts if out_args['failed'] and not keep_missing: with getOutputHandle(seq_file, 'collapse-undetermined', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) \ as missing_handle: for k in search_keys: SeqIO.write(seq_dict[k], missing_handle, out_args['out_type']) if out_args['failed']: # Write duplicate sequences with getOutputHandle(seq_file, 'collapse-duplicate', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) \ as dup_handle: for k in dup_keys: SeqIO.write(seq_dict[k], dup_handle, out_args['out_type']) # Print log log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['SEQUENCES'] = rec_count log['UNIQUE'] = len(uniq_dict) log['DUPLICATE'] = len(dup_keys) log['UNDETERMINED'] = len(search_keys) log['END'] = 'CollapseSeq' printLog(log) # Close file handles pass_file = pass_handle.name if pass_handle is not None: pass_handle.close() if log_handle is not None: log_handle.close() return pass_file
def selectDbFile(db_file, fields, values, logic='any', regex=False, out_args=default_out_args): """ Selects records from a database file Arguments: db_file = the database file name fields = a list of fields to check for selection criteria values = a list of values defining selection targets logic = one of 'any' or 'all' defining whether one or all fields must have a match. regex = if False do exact full string matches; if True allow partial regex matches. out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define string match function if regex: def _match_func(x, patterns): return any([re.search(p, x) for p in patterns]) else: def _match_func(x, patterns): return x in patterns # Define logic function if logic == 'any': _logic_func = any elif logic == 'all': _logic_func = all # Print console log log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'select' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) log['REGEX'] = regex printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Check for selection values in all fields select = _logic_func( [_match_func(rec.get(f, False), values) for f in fields]) # Write sequences if select: pass_count += 1 pass_writer.writerow(rec) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['SELECTED'] = pass_count log['DISCARDED'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def tableLog(record_file, fields, out_file=None, out_args=default_out_args): """ Converts a pRESTO log to a table of annotations Arguments: record_file : the log file name. fields : the list of fields to output. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: the output table file name """ log = OrderedDict() log['START'] = 'ParseLog' log['FILE'] = os.path.basename(record_file) printLog(log) # Open file handles log_handle = open(record_file) if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(record_file, 'table', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over log records start_time = time() record = '' rec_count = pass_count = fail_count = 0 for line in log_handle: if line.strip() == '' and record: # Print progress for previous iteration printCount(rec_count, 1e5, start_time=start_time) # Parse record block rec_count += 1 record_dict = parseLog(record) # Write records if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Empty record string record = '' else: # Append to record record += line else: # Write final record if record: record_dict = parseLog(record) if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Print counts printCount(rec_count, 1e5, start_time, end=True) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseLog' printLog(log) # Close file handles log_handle.close() out_handle.close() return log_handle.name
def sortDbFile(db_file, field, numeric=False, descend=False, out_args=default_out_args): """ Sorts records by values in an annotation field Arguments: db_file = the database filename field = the field name to sort by numeric = if True sort field numerically; if False sort field alphabetically descend = if True sort in descending order; if False sort in ascending order out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUMERIC'] = numeric printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Store all records in a dictionary start_time = time() printMessage("Indexing: Running", start_time=start_time) db_dict = {i: r for i, r in enumerate(db_iter)} result_count = len(db_dict) # Sort db_dict by field values tag_dict = {k: v[field] for k, v in db_dict.items()} if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend) printMessage("Indexing: Done", start_time=start_time, end=True) # Iterate over records start_time = time() rec_count = 0 for key in sorted_keys: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write records pass_writer.writerow(db_dict[key]) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def groupSeqFile(seq_file, field, threshold=None, out_args=default_out_args): """ Divides a sequence file into segments by description tags Arguments: seq_file : filename of the sequence file to split field : The annotation field to split seq_file by threshold : The numerical threshold for group sequences by; if None treat field as textual out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'group' log['FILE'] = os.path.basename(seq_file) log['FIELD'] = field log['THRESHOLD'] = threshold printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Determine total numbers of records rec_count = countSeqFile(seq_file) # Process sequences start_time = time() seq_count = 0 if threshold is None: # Sort records into files based on textual field # Create set of unique field tags temp_iter = readSeqFile(seq_file) tag_list = getAnnotationValues(temp_iter, field, unique=True, delimiter=out_args['delimiter']) if sys.platform != 'win32': import resource # Increase open file handle limit if needed file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0] file_count = len(tag_list) + 256 if file_limit < file_count and file_count <= 8192: #print file_limit, file_count resource.setrlimit(resource.RLIMIT_NOFILE, (file_count, file_count)) elif file_count > 8192: e = '''OS file limit would need to be set to %i. If you are sure you want to do this, then increase the file limit in the OS (via ulimit) and rerun this tool. ''' % file_count printError(dedent(e)) # Create output handles # out_label = '%s=%s' % (field, tag) handles_dict = { tag: getOutputHandle(seq_file, '%s-%s' % (field, tag), out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) for tag in tag_list } # Iterate over sequences for seq in seq_iter: printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write sequences tag = parseAnnotation(seq.description, delimiter=out_args['delimiter'])[field] SeqIO.write(seq, handles_dict[tag], out_args['out_type']) else: # Sort records into files based on numeric threshold threshold = float(threshold) # Create output handles handles_dict = { 'under': getOutputHandle(seq_file, 'under-%.1g' % threshold, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']), 'atleast': getOutputHandle(seq_file, 'atleast-%.1g' % threshold, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) } # Iterate over sequences for seq in seq_iter: printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write sequences tag = parseAnnotation(seq.description, delimiter=out_args['delimiter'])[field] tag = 'under' if float(tag) < threshold else 'atleast' SeqIO.write(seq, handles_dict[tag], out_args['out_type']) # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['SEQUENCES'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'SplitSeq' printLog(log) # Close output file handles for k in handles_dict: handles_dict[k].close() return [handles_dict[k].name for k in handles_dict]
def pairSeq(seq_file_1, seq_file_2, fields_1=None, fields_2=None, coord_type=default_coord_type, out_args=default_out_args): """ Generates consensus sequences Arguments: seq_file_1 = the file containing the grouped sequences and annotations seq_file_2 = the file to assign annotations to from seq_file_1 fields_1 = list of annotations in seq_file_1 records to copy to seq_file_2 records; if None do not copy any annotations fields_2 = list of annotations in seq_file_2 records to copy to seq_file_1 records; if None do not copy any annotations coord_type = the sequence header format out_args = common output argument dictionary from parseCommonArgs Returns: a list of tuples holding successfully paired filenames for (seq_file_1, seq_file_2) """ # Define private functions def _key_func(x): return getCoordKey(x, coord_type=coord_type, delimiter=out_args['delimiter']) log = OrderedDict() log['START'] = 'PairSeq' log['FILE1'] = os.path.basename(seq_file_1) log['FILE2'] = os.path.basename(seq_file_2) log['FIELDS_1'] = ','.join(fields_1) if fields_1 is not None else None log['FIELDS_2'] = ','.join(fields_2) if fields_2 is not None else None log['COORD_TYPE'] = coord_type printLog(log) # Define output type if out_args['out_type'] is None: out_type_1 = getFileType(seq_file_1) out_type_2 = getFileType(seq_file_2) else: out_type_1 = out_type_2 = out_args['out_type'] # Define output name if out_args['out_name'] is None: out_name_1 = out_name_2 = None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] # Open and count files start_time = time() printMessage("Indexing files", start_time=start_time) # Index file 1 seq_count_1 = countSeqFile(seq_file_1) seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func) # Define file 2 iterator seq_count_2 = countSeqFile(seq_file_2) seq_iter_2 = readSeqFile(seq_file_2, index=False) printMessage("Done", start_time=start_time, end=True) # Open output file handles pass_handle_1 = getOutputHandle(seq_file_1, 'pair-pass', out_args['out_dir'], out_name=out_name_1, out_type=out_type_1) pass_handle_2 = getOutputHandle(seq_file_2, 'pair-pass', out_args['out_dir'], out_name=out_name_2, out_type=out_type_2) if out_args['failed']: fail_handle_1 = getOutputHandle(seq_file_1, 'pair-fail', out_dir=out_args['out_dir'], out_name=out_name_1, out_type=out_type_1) fail_handle_2 = getOutputHandle(seq_file_2, 'pair-fail', out_dir=out_args['out_dir'], out_name=out_name_2, out_type=out_type_2) pass_keys = list() # Iterate over pairs and write to output files start_time = time() rec_count = pair_count = 0 for seq_2 in seq_iter_2: # Print progress for previous iteration printProgress(rec_count, seq_count_2, 0.05, start_time) rec_count += 1 # Check for file 2 mate pair in file 1 coord_2 = getCoordKey(seq_2.id, coord_type=coord_type, delimiter=out_args['delimiter']) seq_1 = seq_dict_1.get(coord_2, None) if seq_1 is not None: # Record paired keys pair_count += 1 if fields_1 is not None or fields_2 is not None: ann_1 = parseAnnotation(seq_1.description, delimiter=out_args['delimiter']) ann_2 = parseAnnotation(seq_2.description, delimiter=out_args['delimiter']) # Prepend annotations from seq_1 to seq_2 if fields_1 is not None: copy_ann = OrderedDict([(k, v) for k, v in ann_1.items() \ if k in fields_1]) merge_ann = mergeAnnotation(ann_2, copy_ann, prepend=True, delimiter=out_args['delimiter']) seq_2.id = flattenAnnotation(merge_ann, delimiter=out_args['delimiter']) seq_2.description = '' # Append annotations from seq_2 to seq_1 if fields_2 is not None: copy_ann = OrderedDict([(k, v) for k, v in ann_2.items() \ if k in fields_2]) merge_ann = mergeAnnotation(ann_1, copy_ann, prepend=False, delimiter=out_args['delimiter']) seq_1.id = flattenAnnotation(merge_ann, delimiter=out_args['delimiter']) seq_1.description = '' # Write paired records SeqIO.write(seq_1, pass_handle_1, out_type_1) SeqIO.write(seq_2, pass_handle_2, out_type_2) # Write unpaired file 2 records and updated paired key list for finding unpaired file 1 records if out_args['failed']: if seq_1 is not None: pass_keys.append(coord_2) else: SeqIO.write(seq_2, fail_handle_2, out_type_2) # Print final progress printProgress(rec_count, seq_count_2, 0.05, start_time) # Find and write unpaired file 1 records if out_args['failed']: start_time = time() printMessage("Finding unpaired", start_time=start_time) # Find file 1 unpaired keys pass_keys = set(pass_keys) unpaired = set(seq_dict_1).difference(pass_keys) # Write unpaired file 1 records for k in unpaired: SeqIO.write(seq_dict_1[k], fail_handle_1, out_type_1) printMessage("Done", start_time=start_time, end=True) # Print log log = OrderedDict() log['OUTPUT1'] = os.path.basename(pass_handle_1.name) log['OUTPUT2'] = os.path.basename(pass_handle_2.name) log['SEQUENCES1'] = seq_count_1 log['SEQUENCES2'] = seq_count_2 log['PASS'] = pair_count log['END'] = 'PairSeq' printLog(log) # Close file handles pass_handle_1.close() pass_handle_2.close() return [(pass_handle_1.name, pass_handle_2.name)]
def sampleSeqFile(seq_file, max_count, field=None, values=None, out_args=default_out_args): """ Samples from a sequence file Arguments: seq_file : filename of the sequence file to sample from max_count : a list of the maximum number of sequences to sample field : the annotation field to check for required values values : a list of annotation values that a sample must contain one of out_args : common output argument dictionary from parseCommonArgs Returns: str: output file name """ # Function to sample from a list of sequence indices def _sample_list(n, index_list): max_n = len(index_list) r = random.sample(range(max_n), n) if n < max_n else range(max_n) return [index_list[x] for x in r] # Function to sample from a dictionary of grouped sequence indices def _sample_dict(n, index_dict): sample_list = [] for v in index_dict.values(): max_n = len(v) r = random.sample(range(max_n), n) if n < max_n else range(max_n) sample_list.extend([v[x] for x in r]) return sample_list # Print console log log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'sample' log['FILE'] = os.path.basename(seq_file) log['MAX_COUNTS'] = ','.join([str(x) for x in max_count]) log['FIELD'] = field log['VALUES'] = ','.join(values) if values else None printLog(log) # Read input files and open output files start_time = time() printMessage('Reading files', start_time=start_time, width=25) in_type = getFileType(seq_file) seq_dict = readSeqFile(seq_file, index=True) if out_args['out_type'] is None: out_args['out_type'] = in_type # Generate subset of records if field is not None and values is not None: _sample = _sample_list printMessage('Subsetting by annotation', start_time=start_time, width=25) seq_index = subsetSeqIndex(seq_dict, field, values, delimiter=out_args['delimiter']) elif field is not None and values is None: _sample = _sample_dict printMessage('Indexing by annotation', start_time=start_time, width=25) seq_index = indexSeqSets(seq_dict, field, delimiter=out_args['delimiter']) else: _sample = _sample_list seq_index = [k for k in seq_dict] printMessage('Done', start_time=start_time, end=True, width=25) # Generate sample set for each value in max_count out_files = [] for i, n in enumerate(max_count): start_time = time() printMessage('Sampling n=%i' % n, start_time=start_time, width=25) # Sample from records sample_keys = _sample(n, seq_index) sample_count = len(sample_keys) # Write sampled sequences to files with getOutputHandle(seq_file, 'sample%i-n%i' % (i + 1, sample_count), out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) as out_handle: for k in sample_keys: SeqIO.write(seq_dict[k], out_handle, out_args['out_type']) out_files.append(out_handle.name) printMessage('Done', start_time=start_time, end=True, width=25) # Print log for iteration log = OrderedDict() log['MAX_COUNT'] = n log['SAMPLED'] = sample_count log['OUTPUT'] = os.path.basename(out_files[i]) printLog(log) # Print log log = OrderedDict() log['END'] = 'SplitSeq' printLog(log) return out_files
def collectQueue(alive, result_queue, collect_queue, db_file, out_args, cluster_func=None, cluster_args={}): """ Assembles results from a queue of individual sequence results and manages log/file I/O Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues if False exit process result_queue = a multiprocessing.Queue holding processQueue results collect_queue = a multiprocessing.Queue to store collector return values db_file = the input database file name out_args = common output argument dictionary from parseCommonArgs cluster_func = the function to call for carrying out clustering on distance matrix cluster_args = a dictionary of arguments to pass to cluster_func Returns: None (adds 'log' and 'out_files' to collect_dict) """ # Open output files try: # Count records and define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] result_count = countDbFile(db_file) # Defined successful output handle pass_handle = getOutputHandle(db_file, out_label='clone-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE') # Defined failed alignment output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, out_label='clone-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None fail_writer = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: #sys.stderr.write('Exception in collector file opening step\n') alive.value = False raise # Get results from queue and write to files try: #print 'START COLLECT', alive.value # Iterator over results queue until sentinel object reached start_time = time() rec_count = clone_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break #print "COLLECT", alive.value, result['id'] # Print progress for previous iteration and update record count if rec_count == 0: print('PROGRESS> Assigning clones') printProgress(rec_count, result_count, 0.05, start_time) rec_count += len(result.data) # Write passed and failed records if result: for clone in result.results.values(): clone_count += 1 for i, rec in enumerate(clone): rec.annotations['CLONE'] = clone_count pass_writer.writerow(rec.toDict()) pass_count += 1 result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction) else: for i, rec in enumerate(result.data): if fail_writer is not None: fail_writer.writerow(rec.toDict()) fail_count += 1 result.log['CLONE0-%i' % (i + 1)] = str(rec.junction) # Write log printLog(result.log, handle=log_handle) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(rec_count, result_count, 0.05, start_time) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() # Update return list log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) except: #sys.stderr.write('Exception in collector result processing step\n') alive.value = False raise return None
def downsizeSeqFile(seq_file, max_count, out_args=default_out_args): """ Splits a FASTA/FASTQ file into segments with a limited number of records Arguments: seq_file : filename of the FASTA file to split max_count : number of records in each output file out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'count' log['FILE'] = os.path.basename(seq_file) log['MAX_COUNT'] = max_count printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Determine total numbers of records rec_count = countSeqFile(seq_file) # Loop through iterator writing each record and opening new output handle as needed start_time = time() seq_count, part_num = 0, 1 out_handle = getOutputHandle(seq_file, 'part%06i' % part_num, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files = [out_handle.name] for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, rec_count, 0.05, start_time=start_time) # Update count seq_count += 1 # Write records SeqIO.write(seq, out_handle, out_args['out_type']) # Break if total records reached to avoid extra empty file if seq_count == rec_count: break # Open new file if needed if seq_count % max_count == 0: out_handle.close() part_num += 1 out_handle = getOutputHandle(seq_file, 'part%06i' % part_num, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files.append(out_handle.name) # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, f in enumerate(out_files): log['OUTPUT%i' % (i + 1)] = os.path.basename(f) log['SEQUENCES'] = rec_count log['PARTS'] = len(out_files) log['END'] = 'SplitSeq' printLog(log) # Close file handles out_handle.close() return out_files
def convertHeaders(seq_file, convert_func, convert_args={}, out_args=default_out_args): """ Converts sequence headers to the pRESTO format Arguments: seq_file = the sequence file name convert_func = the function used to convert sequence headers convert_args = a dictionary of arguments to pass to convert_func out_args = common output argument dictionary from parseCommonArgs Returns: the output sequence file name """ # Define subcommand label dictionary cmd_dict = {convertGenericHeader:'generic', convert454Header:'454', convertGenbankHeader:'genbank', convertIlluminaHeader:'illumina', convertIMGTHeader:'imgt', convertSRAHeader:'sra'} log = OrderedDict() log['START'] = 'ConvertHeaders' log['COMMAND'] = cmd_dict[convert_func] log['FILE'] = os.path.basename(seq_file) printLog(log) # Open input file in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Count records result_count = countSeqFile(seq_file) # Open output file handles pass_handle = getOutputHandle(seq_file, 'convert-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) if out_args['failed']: fail_handle = getOutputHandle(seq_file, 'convert-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) else: fail_handle = None # Set additional conversion arguments if convert_func in [convertGenericHeader, convertGenbankHeader]: convert_args.update({'delimiter':out_args['delimiter']}) # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration and update count printProgress(seq_count, result_count, 0.05, start_time) seq_count += 1 # Convert header header = convert_func(seq.description, **convert_args) if header is not None: # Write successfully converted sequences pass_count += 1 seq.id = seq.name = flattenAnnotation(header, out_args['delimiter']) seq.description = '' SeqIO.write(seq, pass_handle, out_args['out_type']) else: fail_count += 1 if fail_handle is not None: # Write successfully unconverted sequences SeqIO.write(seq, fail_handle, out_args['out_type']) # Print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ConvertHeaders' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() return pass_handle.name
def samplePairSeqFile(seq_file_1, seq_file_2, max_count, field=None, values=None, coord_type=default_coord, out_args=default_out_args): """ Samples from paired-end sequence files Arguments: seq_file_1 : filename of the first paired-end sequence file seq_file_2 : filename of the second paired-end sequence file max_count : a list of the maximum number of sequences to sample field : the annotation field to check for required values values : a list of annotation values that a sample must contain one of coord_type : the sequence header format out_args : common output argument dictionary from parseCommonArgs Returns: list: seq_file_1 and seq_file_2 output file names """ # Sequence index key function def _key_func(x): return getCoordKey(x, coord_type=coord_type, delimiter=out_args['delimiter']) # Function to sample from two lists of sequence indices def _sample_list(n, index_1, index_2): key_set = set(index_1).intersection(index_2) max_n = len(key_set) return random.sample(key_set, min(n, max_n)) # Function to sample from two dictionaries of grouped sequence indices def _sample_dict(n, index_1, index_2): group_set = set(index_1.keys()).intersection(index_2.keys()) sample_list = [] for k in group_set: key_set = set(index_1[k]).intersection(index_2[k]) max_n = len(key_set) sample_list.extend(random.sample(key_set, min(n, max_n))) return sample_list # Print console log log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'samplepair' log['FILE1'] = os.path.basename(seq_file_1) log['FILE2'] = os.path.basename(seq_file_2) log['MAX_COUNTS'] = ','.join([str(x) for x in max_count]) log['FIELD'] = field log['VALUES'] = ','.join(values) if values else None printLog(log) # Define output type in_type_1 = getFileType(seq_file_1) in_type_2 = getFileType(seq_file_2) if out_args['out_type'] is None: out_type_1 = in_type_1 out_type_2 = in_type_2 else: out_type_1 = out_type_2 = out_args['out_type'] # Define output name if out_args['out_name'] is None: out_name_1 = out_name_2 = None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] # Index input files start_time = time() printMessage('Reading files', start_time=start_time, width=25) seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func) seq_dict_2 = readSeqFile(seq_file_2, index=True, key_func=_key_func) # Subset keys to those meeting field/value criteria if field is not None and values is not None: _sample = _sample_list printMessage('Subsetting by annotation', start_time=start_time, width=25) seq_index_1 = subsetSeqIndex(seq_dict_1, field, values, delimiter=out_args['delimiter']) seq_index_2 = subsetSeqIndex(seq_dict_2, field, values, delimiter=out_args['delimiter']) elif field is not None and values is None: _sample = _sample_dict printMessage('Indexing by annotation', start_time=start_time, width=25) seq_index_1 = indexSeqSets(seq_dict_1, field, delimiter=out_args['delimiter']) seq_index_2 = indexSeqSets(seq_dict_2, field, delimiter=out_args['delimiter']) else: _sample = _sample_list seq_index_1 = list(seq_dict_1.keys()) seq_index_2 = list(seq_dict_2.keys()) printMessage('Done', start_time=start_time, end=True, width=25) # Generate sample set for each value in max_count out_files = [] for i, n in enumerate(max_count): start_time = time() printMessage('Sampling n=%i' % n, start_time=start_time, width=25) # Sample sample_keys = _sample(n, seq_index_1, seq_index_2) sample_count = len(sample_keys) # Open file handles out_handle_1 = getOutputHandle(seq_file_1, 'sample%i-n%i' % (i + 1, sample_count), out_dir=out_args['out_dir'], out_name=out_name_1, out_type=out_type_1) out_handle_2 = getOutputHandle(seq_file_2, 'sample%i-n%i' % (i + 1, sample_count), out_dir=out_args['out_dir'], out_name=out_name_2, out_type=out_type_2) out_files.append((out_handle_1.name, out_handle_2.name)) for k in sample_keys: SeqIO.write(seq_dict_1[k], out_handle_1, out_type_1) SeqIO.write(seq_dict_2[k], out_handle_2, out_type_2) printMessage('Done', start_time=start_time, end=True, width=25) # Print log for iteration log = OrderedDict() log['MAX_COUNT'] = n log['SAMPLED'] = sample_count log['OUTPUT1'] = os.path.basename(out_files[i][0]) log['OUTPUT2'] = os.path.basename(out_files[i][1]) printLog(log) # Close file handles out_handle_1.close() out_handle_2.close() # Print log log = OrderedDict() log['END'] = 'SplitSeq' printLog(log) return out_files
def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sequences meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles out_type = 'fasta' db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Get SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter']) # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, out_type) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def sortSeqFile(seq_file, field, numeric=False, max_count=None, out_args=default_out_args): """ Sorts a sequence file by annotation fields Arguments: seq_file : filename of the sequence file to split field : position of field in sequence description to split by numeric : if True sort field numerically; if False sort field alphabetically max_count : maximum number of records in each output file if None do not create multiple files out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(seq_file) log['FIELD'] = field log['NUMERIC'] = numeric log['MAX_COUNT'] = max_count printLog(log) # Open file handles in_type = getFileType(seq_file) seq_dict = readSeqFile(seq_file, index=True) if out_args['out_type'] is None: out_args['out_type'] = in_type # Get annotations and sort seq_dict by annotation values tag_dict = { k: parseAnnotation(seq_dict[k].description, delimiter=out_args['delimiter'])[field] for k in seq_dict } if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get) # Determine total numbers of records rec_count = len(seq_dict) if max_count >= rec_count: max_count = None # Open initial output file handles file_count = 1 if max_count is None: out_label = 'sorted' else: out_label = 'sorted-part%06i' % file_count out_handle = getOutputHandle(seq_file, out_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files = [out_handle.name] # Loop through sorted sequence dictionary keys start_time = time() last_tag = None saved_keys = [] seq_count = chunk_count = 0 for key in sorted_keys: # Print progress for previous iteration and update count printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write saved group of sequences when tag changes if last_tag is not None and tag_dict[key] != last_tag: # Open new output file if needed if max_count is not None and chunk_count + len( saved_keys) > max_count: # Update partition counts file_count += 1 chunk_count = 0 # Open new file handle out_handle.close() out_handle = getOutputHandle(seq_file, 'sorted-part%06i' % file_count, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Append output file name to out_files out_files.append(out_handle.name) # Write saved sequences for k in saved_keys: chunk_count += 1 SeqIO.write(seq_dict[k], out_handle, out_args['out_type']) # Reset saved keys to current key only saved_keys = [key] else: # Update list of saved keys if tag is unchanged saved_keys.append(key) # Check if total records reached, write all saved keys, and exit loop if seq_count == rec_count: for k in saved_keys: chunk_count += 1 SeqIO.write(seq_dict[k], out_handle, out_args['out_type']) out_handle.close() break # Update tag tracker last_tag = tag_dict[key] # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, f in enumerate(out_files): log['OUTPUT%i' % (i + 1)] = os.path.basename(f) log['SEQUENCES'] = seq_count log['PARTS'] = len(out_files) log['END'] = 'SplitSeq' printLog(log) # Close file handles out_handle.close() return out_files
def selectSeqFile(seq_file, field, value_list=None, value_file=None, negate=False, out_file=None, out_args=default_out_args): """ Select from a sequence file Arguments: seq_file : filename of the sequence file to sample from. field : the annotation field to check for required values. value_list : a list of annotation values that a sample must contain one of. value_file : a tab delimited file containing values to select. negate : if True select entires that do not contain the specific values. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: output file name. """ # Reads value_file def _read_file(value_file, field): field_list = [] try: with open(value_file, 'rt') as handle: reader_dict = csv.DictReader(handle, dialect='excel-tab') for row in reader_dict: field_list.append(row[field]) except IOError: printError('File %s cannot be read.' % value_file) except: printError('File %s is invalid.' % value_file) return field_list # Print console log log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'select' log['FILE'] = os.path.basename(seq_file) log['FIELD'] = field if value_list is not None: log['VALUE_LIST'] = ','.join([str(x) for x in value_list]) if value_file is not None: log['VALUE_FILE'] = os.path.basename(value_file) log['NOT'] = negate printLog(log) # Read value_file if value_list is not None and value_file is not None: printError('Specify only one of value_list and value_file.') elif value_file is not None: value_list = _read_file(value_file, field) # Read sequence file in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Output output handle if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(seq_file, 'selected', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Generate subset of records start_time = time() pass_count, fail_count, rec_count = 0, 0, 0 value_set = set(value_list) for rec in seq_iter: printCount(rec_count, 1e5, start_time=start_time) rec_count += 1 # Parse annotations into a list of values ann = parseAnnotation(rec.description, delimiter=out_args['delimiter'])[field] ann = ann.split(out_args['delimiter'][2]) # Write if xor(negate, not value_set.isdisjoint(ann)): # Write SeqIO.write(rec, out_handle, out_args['out_type']) pass_count += 1 else: fail_count += 1 printCount(rec_count, 1e5, start_time=start_time, end=True) # Print log log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'SplitSeq' printLog(log) return out_handle.name
def modifyHeaders(seq_file, modify_func, modify_args, out_args=default_out_args): """ Modifies sequence headers Arguments: seq_file = the sequence file name modify_func = the function defining the modification operation modify_args = a dictionary of arguments to pass to modify_func out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define subcommand label dictionary cmd_dict = {addHeader: 'add', copyHeader: 'copy', collapseHeader: 'collapse', deleteHeader: 'delete', expandHeader: 'expand', renameHeader: 'rename'} # Print parameter info log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__) log['FILE'] = os.path.basename(seq_file) for k in sorted(modify_args): v = modify_args[k] log[k.upper()] = ','.join(v) if isinstance(v, list) else v printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Count records result_count = countSeqFile(seq_file) # Iterate over sequences start_time = time() seq_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time) #Update counts seq_count += 1 # Modify header header = parseAnnotation(seq.description, delimiter=out_args['delimiter']) header = modify_func(header, delimiter=out_args['delimiter'], **modify_args) # Write new sequence seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter']) seq.description = '' SeqIO.write(seq, out_handle, out_args['out_type']) # print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args): """ Renames fields in a database file Arguments: db_file = the database file name fields = a list of fields to rename values = a list of new names for fields out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'rename' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['NAMES'] = ','.join(names) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Get header and rename fields header = (readDbFile(db_file, ig=False)).fieldnames for f, n in zip(fields, names): i = header.index(f) header[i] = n # Open writer and write new header # TODO: should modify getDbWriter to take a list of fields pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab') pass_writer.writeheader() # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore # Rename fields for f, n in zip(fields, names): rec[n] = rec.pop(f) # Write pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def tableHeaders(seq_file, fields, out_args=default_out_args): """ Builds a table of sequence header annotations Arguments: seq_file = the sequence file name fields = the list of fields to output out_args = common output argument dictionary from parseCommonArgs Returns: the output table file name """ log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(seq_file) printLog(log) # Open file handles seq_iter = readSeqFile(seq_file) out_handle = getOutputHandle(seq_file, out_label='headers', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Count records result_count = countSeqFile(seq_file) # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time) # Get annotations seq_count += 1 ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter']) # Write records if ann: pass_count += 1 out_writer.writerow(ann) else: fail_count += 1 # Print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args): """ Divides a tab-delimited database file into segments by description tags Arguments: db_file = filename of the tab-delimited database file to split field = the field name by which to split db_file num_split = the numerical threshold by which to group sequences; if None treat field as textual out_args = common output argument dictionary from parseCommonArgs Returns: a list of output file names """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'split' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUM_SPLIT'] = num_split printLog(log) # Open IgRecord reader iter object reader = readDbFile(db_file, ig=False) # Determine total numbers of records rec_count = countDbFile(db_file) start_time = time() count = 0 # Sort records into files based on textual field if num_split is None: # Create set of unique field tags tmp_iter = readDbFile(db_file, ig=False) tag_list = list(set([row[field] for row in tmp_iter])) # Forbidden characters in filename and replacements noGood = {'\/':'f','\\':'b','?':'q','\%':'p','*':'s',':':'c', '\|':'pi','\"':'dq','\'':'sq','<':'gt','>':'lt',' ':'_'} # Replace forbidden characters in tag_list tag_dict = {} for tag in tag_list: for c,r in noGood.items(): tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \ if c in tag else tag_dict.get(tag, tag)) # Create output handles handles_dict = {tag:getOutputHandle(db_file, '%s-%s' % (field, label), out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir']) for tag, label in tag_dict.items()} # Create Db writer instances writers_dict = {tag:getDbWriter(handles_dict[tag], db_file) for tag in tag_dict} # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 # Write row to appropriate file tag = row[field] writers_dict[tag].writerow(row) # Sort records into files based on numeric num_split else: num_split = float(num_split) # Create output handles handles_dict = {'under':getOutputHandle(db_file, 'under-%.1f' % num_split, out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir']), 'atleast':getOutputHandle(db_file, 'atleast-%.1f' % num_split, out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir'])} # Create Db writer instances writers_dict = {'under':getDbWriter(handles_dict['under'], db_file), 'atleast':getDbWriter(handles_dict['atleast'], db_file)} # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 tag = row[field] tag = 'under' if float(tag) < num_split else 'atleast' writers_dict[tag].writerow(row) # Write log printProgress(count, rec_count, 0.05, start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['RECORDS'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'ParseDb' printLog(log) # Close output file handles for t in handles_dict: handles_dict[t].close() return [handles_dict[t].name for t in handles_dict]
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Write germline sequences to tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'False' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader references = readRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS'] # Create output file handle and Db writer pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None fail_writer = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) pass_count = fail_count = 0 # Iterate over rows for i, row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) result_log, germlines = joinGermline(row, references, seq_field=seq_field, v_field=v_field, germ_types=germ_types) # Add germline field(s) to dictionary if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full'] if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask'] if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly'] if 'regions' in germ_types: row['GERMLINE_REGIONS'] = germlines['regions'] # Write row to pass or fail file if 'ERROR' in result_log: fail_count += 1 if fail_writer is not None: fail_writer.writerow(row) else: result_log['SEQUENCE'] = row[seq_field] result_log['GERMLINE'] = germlines['full'] result_log['REGIONS'] = germlines['regions'] pass_count += 1 pass_writer.writerow(row) printLog(result_log, handle=log_handle) # Print log printProgress(i + 1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def updateDbFile(db_file, field, values, updates, out_args=default_out_args): """ Updates field and value pairs to a database file Arguments: db_file = the database file name field = the field to update values = a list of values to specifying which rows to update updates = a list of values to update each value with out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'update' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['VALUES'] = ','.join(values) log['UPDATES'] = ','.join(updates) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Updated values if found for x, y in zip(values, updates): if rec[field] == x: rec[field] = y pass_count += 1 # Write records pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['UPDATED'] = pass_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args, cluster_func, cluster_args): """ Assembles results from a queue of individual sequence results and manages log/file I/O Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues if False exit process result_queue = a multiprocessing.Queue holding processQueue results collect_queue = a multiprocessing.Queue to store collector return values db_file = the input database file name out_args = common output argument dictionary from parseCommonArgs cluster_func = the function to call for carrying out clustering on distance matrix cluster_args = a dictionary of arguments to pass to cluster_func Returns: None (adds 'log' and 'out_files' to collect_dict) """ # Open output files try: # Iterate over Ig records to count and order by junction length result_count = 0 records = {} # print 'Reading file...' db_iter = readDbFile(db_file) for rec in db_iter: records[rec.id] = rec result_count += 1 records = OrderedDict( sorted(list(records.items()), key=lambda i: i[1].junction_length)) # Define empty matrix to store assembled results dist_mat = np.zeros((result_count, result_count)) # Count records and define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined successful output handle pass_handle = getOutputHandle(db_file, out_label='clone-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE') # Defined failed cloning output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, out_label='clone-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None fail_writer = None # Open log file if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() row_count = rec_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration if row_count == 0: print('PROGRESS> Assigning clones') printProgress(row_count, result_count, 0.05, start_time) # Update counts for iteration row_count += 1 rec_count += len(result) # Add result row to distance matrix if result: dist_mat[list(range(result_count - len(result), result_count)), result_count - len(result)] = result.results else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Calculate linkage and carry out clustering # print dist_mat clusters = cluster_func(dist_mat, ** cluster_args) if dist_mat is not None else None clones = {} # print clusters for i, c in enumerate(clusters): clones.setdefault(c, []).append(records[list(records.keys())[i]]) # Write passed and failed records clone_count = pass_count = fail_count = 0 if clones: for clone in clones.values(): clone_count += 1 for i, rec in enumerate(clone): rec.annotations['CLONE'] = clone_count pass_writer.writerow(rec.toDict()) pass_count += 1 #result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction) else: for i, rec in enumerate(result.data): fail_writer.writerow(rec.toDict()) fail_count += 1 #result.log['CLONE0-%i' % (i + 1)] = str(rec.junction) # Print final progress printProgress(row_count, result_count, 0.05, start_time) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() # Update return list log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log': log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) except: alive.value = False raise return None