def addDbFile(db_file, fields, values, out_args=default_out_args): """ Adds field and value pairs to a database file Arguments: db_file = the database file name fields = a list of fields to add values = a list of values to assign to all rows of each field out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields) # Count records result_count = countDbFile(db_file) # Define fields and values to append add_dict = {k:v for k,v in zip(fields, values) if k not in db_iter.fieldnames} # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write updated row rec.update(add_dict) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args): """ Adds an index column to a database file Arguments: db_file = the database file name field = the name of the index field to add out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'index' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=field) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Add count and write updated row rec.update({field: rec_count}) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def dropDbFile(db_file, fields, out_args=default_out_args): """ Deletes entire fields from a database file Arguments: db_file = the database file name fields = a list of fields to drop out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write row pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_args=default_out_args): """ Adds an index column to a database file Arguments: db_file = the database file name field = the name of the index field to add out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'index' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=field) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Add count and write updated row rec.update({field:rec_count}) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def dropDbFile(db_file, fields, out_file=None, out_args=default_out_args): """ Deletes entire fields from a database file Arguments: db_file : the database file name. fields : a list of fields to drop. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) __, __, out_args['out_type'] = splitName(db_file) # Exclude dropped field from output out_fields = [f for f in db_iter.fields if f not in fields] # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Write row pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def addDbFile(db_file, fields, values, out_file=None, out_args=default_out_args): """ Adds field and value pairs to a database file Arguments: db_file : the database file name. fields : a list of fields to add. values : a list of values to assign to all rows of each field. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open inut db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) __, __, out_args['out_type'] = splitName(db_file) # Add fields out_fields = list(db_iter.fields) out_fields.extend(fields) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Define fields and values to append add_dict = { k: v for k, v in zip(fields, values) if k not in db_iter.fields } # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Write updated row rec.update(add_dict) pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def convertToFasta(db_file, id_field=default_id_field, seq_field=default_seq_field, meta_fields=None, out_file=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file : the database file name. id_field : the field containing identifiers. seq_field : the field containing sequences. meta_fields : a list of fields to add to sequence annotations. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ConvertDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open input out_type = 'fasta' db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) result_count = countDbFile(db_file) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Iterate over records start_time = time() rec_count, pass_count, fail_count = 0, 0, 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Get SeqRecord seq = buildSeqRecord(rec, id_field, seq_field, meta_fields) # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, out_type) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ConvertDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def collectPairQueue(alive, result_queue, collect_queue, result_count, seq_file_1, seq_file_2, out_args): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue = a multiprocessing.Queue holding worker results collect_queue = a multiprocessing.Queue holding collector return values result_count = the number of expected assembled sequences seq_file_1 = the first sequence file name seq_file_2 = the second sequence file name out_args = common output argument dictionary from parseCommonArgs Returns: None (adds a dictionary of {log: log object, out_files: output file names} to collect_queue) """ try: # Count records and define output format out_type = getFileType(seq_file_1) if out_args['out_type'] is None \ else out_args['out_type'] # Defined valid assembly output handle pass_handle = getOutputHandle(seq_file_1, 'assemble-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Defined failed assembly output handles if out_args['failed']: # Define output name if out_args['out_name'] is None: out_name_1 = out_name_2 = None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] fail_handle_1 = getOutputHandle(seq_file_1, 'assemble-fail', out_dir=out_args['out_dir'], out_name=out_name_1, out_type=out_type) fail_handle_2 = getOutputHandle(seq_file_2, 'assemble-fail', out_dir=out_args['out_dir'], out_name=out_name_2, out_type=out_type) else: fail_handle_1 = None fail_handle_2 = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() iter_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(iter_count, result_count, 0.05, start_time) # Update counts for iteration iter_count += 1 # Write log printLog(result.log, handle=log_handle) # Write assembled sequences if result: pass_count += 1 SeqIO.write(result.results, pass_handle, out_type) else: fail_count += 1 if fail_handle_1 is not None and fail_handle_2 is not None: SeqIO.write(result.data[0], fail_handle_1, out_type) SeqIO.write(result.data[1], fail_handle_2, out_type) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(iter_count, result_count, 0.05, start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['PAIRS'] = iter_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) # Close file handles pass_handle.close() if fail_handle_1 is not None: fail_handle_1.close() if fail_handle_2 is not None: fail_handle_2.close() if log_handle is not None: log_handle.close() except: alive.value = False raise return None
def selectDbFile(db_file, fields, values, logic='any', regex=False, out_args=default_out_args): """ Selects records from a database file Arguments: db_file = the database file name fields = a list of fields to check for selection criteria values = a list of values defining selection targets logic = one of 'any' or 'all' defining whether one or all fields must have a match. regex = if False do exact full string matches; if True allow partial regex matches. out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define string match function if regex: def _match_func(x, patterns): return any([re.search(p, x) for p in patterns]) else: def _match_func(x, patterns): return x in patterns # Define logic function if logic == 'any': _logic_func = any elif logic == 'all': _logic_func = all # Print console log log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'select' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) log['REGEX'] =regex printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Check for selection values in all fields select = _logic_func([_match_func(rec.get(f, False), values) for f in fields]) # Write sequences if select: pass_count += 1 pass_writer.writerow(rec) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['SELECTED'] = pass_count log['DISCARDED'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def collectDbQueue(alive, result_queue, collect_queue, db_file, task_label, out_args, add_fields=None): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive : multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue : multiprocessing.Queue holding worker results collect_queue : multiprocessing.Queue to store collector return values db_file : Database file name task_label : Task label used to tag the output files out_args : Common output argument dictionary from parseCommonArgs add_fields : List of fields added to the writer not present in the in_file; if None do not add fields Returns: None : Adds a dictionary with key value pairs to collect_queue containing 'log' defining a log object, 'out_files' defining the output file names """ try: result_count = countDbFile(db_file) # Define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined valid alignment output handle pass_handle = getOutputHandle(db_file, '%s-pass' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) # Defined failed alignment output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, '%s-fail' % task_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() set_count = rec_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(pass_count, result_count, 0.05, start_time) # Update counts for current iteration set_count += 1 rec_count += result.data_count # Write log printLog(result.log, handle=log_handle) # Write alignments if result: pass_count += result.data_count for rec in result.results: pass_writer.writerow(rec.toDict()) else: fail_count += result.data_count if fail_handle is not None: for rec in result.data: pass_writer.writerow(rec.toDict()) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(pass_count, result_count, 0.05, start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GROUPS'] = set_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() except: alive.value = False raise return None
def mergeDbFiles(db_files, drop=False, out_file=None, out_args=default_out_args): """ Updates field and value pairs to a database file Arguments: db_files : list of database file names. drop : if True drop columns not present in all files. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'merge' log['FILES'] = ','.join([os.path.basename(f) for f in db_files]) log['DROP'] = drop printLog(log) # Open input db_handles = [open(f, 'rt') for f in db_files] db_iters = [TSVReader(x) for x in db_handles] result_count = sum([countDbFile(f) for f in db_files]) # Define output fields field_list = [x.fields for x in db_iters] if drop: field_set = set.intersection(*map(set, field_list)) else: field_set = set.union(*map(set, field_list)) field_order = OrderedDict([(f, None) for f in chain(*field_list)]) out_fields = [f for f in field_order if f in field_set] # Open output file if out_file is not None: pass_handle = open(out_file, 'w') else: __, __, out_args['out_type'] = splitName(db_files[0]) pass_handle = getOutputHandle(db_files[0], out_label='parse-merge', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Iterate over records start_time = time() rec_count = 0 for db in db_iters: for rec in db: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Write records pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() for x in db_handles: x.close() return pass_handle.name
def downsizeSeqFile(seq_file, max_count, out_args=default_out_args): """ Splits a FASTA/FASTQ file into segments with a limited number of records Arguments: seq_file : filename of the FASTA file to split max_count : number of records in each output file out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'count' log['FILE'] = os.path.basename(seq_file) log['MAX_COUNT'] = max_count printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Determine total numbers of records rec_count = countSeqFile(seq_file) # Loop through iterator writing each record and opening new output handle as needed start_time = time() seq_count, part_num = 0, 1 out_handle = getOutputHandle(seq_file, 'part%06i' % part_num, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files = [out_handle.name] for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, rec_count, 0.05, start_time=start_time) # Update count seq_count += 1 # Write records SeqIO.write(seq, out_handle, out_args['out_type']) # Break if total records reached to avoid extra empty file if seq_count == rec_count: break # Open new file if needed if seq_count % max_count == 0: out_handle.close() part_num += 1 out_handle = getOutputHandle(seq_file, 'part%06i' % part_num, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files.append(out_handle.name) # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, f in enumerate(out_files): log['OUTPUT%i' % (i + 1)] = os.path.basename(f) log['SEQUENCES'] = rec_count log['PARTS'] = len(out_files) log['END'] = 'SplitSeq' printLog(log) # Close file handles out_handle.close() return out_files
def groupSeqFile(seq_file, field, threshold=None, out_args=default_out_args): """ Divides a sequence file into segments by description tags Arguments: seq_file : filename of the sequence file to split field : The annotation field to split seq_file by threshold : The numerical threshold for group sequences by; if None treat field as textual out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'group' log['FILE'] = os.path.basename(seq_file) log['FIELD'] = field log['THRESHOLD'] = threshold printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Determine total numbers of records rec_count = countSeqFile(seq_file) # Process sequences start_time = time() seq_count = 0 if threshold is None: # Sort records into files based on textual field # Create set of unique field tags temp_iter = readSeqFile(seq_file) tag_list = getAnnotationValues(temp_iter, field, unique=True, delimiter=out_args['delimiter']) if sys.platform != 'win32': import resource # Increase open file handle limit if needed file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0] file_count = len(tag_list) + 256 if file_limit < file_count and file_count <= 8192: #print file_limit, file_count resource.setrlimit(resource.RLIMIT_NOFILE, (file_count, file_count)) elif file_count > 8192: e = '''OS file limit would need to be set to %i. If you are sure you want to do this, then increase the file limit in the OS (via ulimit) and rerun this tool. ''' % file_count printError(dedent(e)) # Create output handles # out_label = '%s=%s' % (field, tag) handles_dict = { tag: getOutputHandle(seq_file, '%s-%s' % (field, tag), out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) for tag in tag_list } # Iterate over sequences for seq in seq_iter: printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write sequences tag = parseAnnotation(seq.description, delimiter=out_args['delimiter'])[field] SeqIO.write(seq, handles_dict[tag], out_args['out_type']) else: # Sort records into files based on numeric threshold threshold = float(threshold) # Create output handles handles_dict = { 'under': getOutputHandle(seq_file, 'under-%.1g' % threshold, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']), 'atleast': getOutputHandle(seq_file, 'atleast-%.1g' % threshold, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) } # Iterate over sequences for seq in seq_iter: printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write sequences tag = parseAnnotation(seq.description, delimiter=out_args['delimiter'])[field] tag = 'under' if float(tag) < threshold else 'atleast' SeqIO.write(seq, handles_dict[tag], out_args['out_type']) # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['SEQUENCES'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'SplitSeq' printLog(log) # Close output file handles for k in handles_dict: handles_dict[k].close() return [handles_dict[k].name for k in handles_dict]
def collectPairQueue(alive, result_queue, collect_queue, seq_file_1, seq_file_2, label, out_file=None, out_args=default_out_args): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive : a multiprocessing.Value boolean controlling whether processing continues; when False function returns. result_queue : a multiprocessing.Queue holding worker results. collect_queue : a multiprocessing.Queue holding collector return values. seq_file_1 : the first sequence file name. seq_file_2 : the second sequence file name. label : task label used to tag the output files. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: None: adds a dictionary of {log: log object, out_files: output file names} to collect_queue. """ # Define output format out_type = getFileType(seq_file_1) if out_args['out_type'] is None \ else out_args['out_type'] # Define output names if out_args['out_name'] is None: out_name_1, out_name_2 = None, None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] # Wrapper for opening handles and writers def _open(x, in_file, out_name, out_file=out_file): if out_file is not None and x == 'pass': handle = open(out_file, 'w') else: handle = getOutputHandle(in_file, out_label='%s-%s' % (label, x), out_dir=out_args['out_dir'], out_name=out_name, out_type=out_type) return handle try: # Count input size result_count = countSeqFile(seq_file_1) # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Initialize file handles pass_handle, fail_handle_1, fail_handle_2 = None, None, None # Iterator over results queue until sentinel object reached start_time = time() iter_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(iter_count, result_count, 0.05, start_time=start_time) # Update counts for iteration iter_count += 1 # Write log printLog(result.log, handle=log_handle) # Write assembled sequences if result: pass_count += 1 try: SeqIO.write(result.results, pass_handle, out_type) except AttributeError: # Open pass file pass_handle = _open('pass', seq_file_1, out_args['out_name']) SeqIO.write(result.results, pass_handle, out_type) else: fail_count += 1 if out_args['failed']: try: SeqIO.write(result.data[0], fail_handle_1, out_type) SeqIO.write(result.data[1], fail_handle_2, out_type) except AttributeError: # Open fail file fail_handle_1 = _open('fail', seq_file_1, out_name_1) fail_handle_2 = _open('fail', seq_file_2, out_name_2) SeqIO.write(result.data[0], fail_handle_1, out_type) SeqIO.write(result.data[1], fail_handle_2, out_type) else: sys.stderr.write('PID %s> Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(iter_count, result_count, 0.05, start_time=start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename( pass_handle.name) if pass_handle is not None else None log['PAIRS'] = iter_count log['PASS'] = pass_count log['FAIL'] = fail_count # Close file handles and generate return data # collect_dict = {'log': log, 'pass': None, 'fail': None} collect_dict = {'log': log, 'out_files': []} if pass_handle is not None: # collect_dict['pass'] = pass_handle.name collect_dict['out_files'].append(pass_handle.name) pass_handle.close() if fail_handle_1 is not None: # collect_dict['fail'] = fail_handle.name collect_dict['out_files'].append(fail_handle_1.name) fail_handle_1.close() if fail_handle_2 is not None: # collect_dict['fail'] = fail_handle.name collect_dict['out_files'].append(fail_handle_2.name) fail_handle_2.close() if log_handle is not None: log_handle.close() collect_queue.put(collect_dict) except: alive.value = False raise return None
def collectSeqQueue(alive, result_queue, collect_queue, seq_file, label, index_field=None, out_file=None, out_args=default_out_args): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive : a multiprocessing.Value boolean controlling whether processing continues; when False function returns. result_queue : Multiprocessing.Queue holding worker results. collect_queue : Multiprocessing.Queue to store collector return values. seq_file : sample sequence file name. label : task label used to tag the output files. out_file : output file name. Automatically generated from the input file if None. out_args : Common output argument dictionary from parseCommonArgs. index_field : Field defining set membership for sequence sets if None data queue contained individual records. Returns: None: Adds a dictionary with key value pairs to collect_queue containing 'log' defining a log object, 'out_files' defining the output file names """ # Define output format out_type = getFileType(seq_file) if out_args['out_type'] is None \ else out_args['out_type'] # Wrapper for opening handles and writers def _open(x, label=label, out_file=out_file): if out_file is not None and x == 'pass': handle = open(out_file, 'w') else: handle = getOutputHandle(seq_file, out_label='%s-%s' % (label, x), out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) return handle try: # Count records if index_field is None: result_count = countSeqFile(seq_file) else: result_count = countSeqSets(seq_file, index_field, out_args['delimiter']) # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Initialize output handles pass_handle, fail_handle = None, None # Iterator over results queue until sentinel object reached start_time = time() set_count = seq_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(set_count, result_count, 0.05, start_time=start_time) # Update counts for current iteration set_count += 1 seq_count += result.data_count # Write log printLog(result.log, handle=log_handle) # Write records if result: pass_count += 1 try: SeqIO.write(result.results, pass_handle, out_type) except AttributeError: # Open pass file pass_handle = _open('pass') SeqIO.write(result.results, pass_handle, out_type) else: fail_count += 1 if out_args['failed']: try: SeqIO.write(result.data, fail_handle, out_type) except AttributeError: # Open fail file fail_handle = _open('fail') SeqIO.write(result.data, fail_handle, out_type) else: sys.stderr.write('PID %s> Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(set_count, result_count, 0.05, start_time=start_time) # Update return values log = OrderedDict() log['OUTPUT'] = os.path.basename( pass_handle.name) if pass_handle is not None else None log['SEQUENCES'] = seq_count if index_field is not None: log['SETS'] = set_count log['PASS'] = pass_count log['FAIL'] = fail_count # Close file handles and generate return data #collect_dict = {'log': log, 'pass': None, 'fail': None} collect_dict = {'log': log, 'out_files': []} if pass_handle is not None: #collect_dict['pass'] = pass_handle.name collect_dict['out_files'].append(pass_handle.name) pass_handle.close() if fail_handle is not None: #collect_dict['fail'] = fail_handle.name collect_dict['out_files'].append(fail_handle.name) fail_handle.close() if log_handle is not None: log_handle.close() collect_queue.put(collect_dict) except: alive.value = False raise return None
def modifyHeaders(seq_file, modify_func, modify_args, out_file=None, out_args=default_out_args): """ Modifies sequence headers Arguments: seq_file : the sequence file name. modify_func : the function defining the modification operation. modify_args : a dictionary of arguments to pass to modify_func. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: output file name. """ # Define subcommand label dictionary cmd_dict = {addHeader: 'add', copyHeader: 'copy', collapseHeader: 'collapse', deleteHeader: 'delete', expandHeader: 'expand', renameHeader: 'rename'} # Print parameter info log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__) log['FILE'] = os.path.basename(seq_file) for k in sorted(modify_args): v = modify_args[k] log[k.upper()] = ','.join(v) if isinstance(v, list) else v printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Count records result_count = countSeqFile(seq_file) # Iterate over sequences start_time = time() seq_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time=start_time) #Update counts seq_count += 1 # Modify header header = parseAnnotation(seq.description, delimiter=out_args['delimiter']) header = modify_func(header, delimiter=out_args['delimiter'], **modify_args) # Write new sequence seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter']) seq.description = '' SeqIO.write(seq, out_handle, out_args['out_type']) # print counts printProgress(seq_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def tableHeaders(seq_file, fields, out_file=None, out_args=default_out_args): """ Builds a table of sequence header annotations Arguments: seq_file : the sequence file name. fields : the list of fields to output. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str: output table file name """ log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(seq_file) printLog(log) # Open file handles seq_iter = readSeqFile(seq_file) if out_file is not None: out_handle = open(out_file, 'w') else: out_handle = getOutputHandle(seq_file, 'headers', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Count records result_count = countSeqFile(seq_file) # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time=start_time) # Get annotations seq_count += 1 ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter']) # Write records if ann: pass_count += 1 out_writer.writerow(ann) else: fail_count += 1 # Print counts printProgress(seq_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def renameDbFile(db_file, fields, names, out_file=None, out_args=default_out_args): """ Renames fields in a database file Arguments: db_file : the database file name. fields : a list of fields to rename. values : a list of new names for fields. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'rename' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['NAMES'] = ','.join(names) printLog(log) # Open file handles db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) __, __, out_args['out_type'] = splitName(db_file) # Get header and rename fields out_fields = list(db_iter.fields) for f, n in zip(fields, names): i = out_fields.index(f) out_fields[i] = n # Open writer if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # TODO: repeating renaming is unnecessary. # Rename fields for f, n in zip(fields, names): rec[n] = rec.pop(f) # Write pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def sortDbFile(db_file, field, numeric=False, descend=False, out_file=None, out_args=default_out_args): """ Sorts records by values in an annotation field Arguments: db_file : the database filename field : the field name to sort by numeric : if True sort field numerically; if False sort field alphabetically descend : if True sort in descending order; if False sort in ascending order out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs Returns: str : output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUMERIC'] = numeric printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) out_fields = db_iter.fields __, __, out_args['out_type'] = splitName(db_file) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Store all records in a dictionary start_time = time() printMessage("Indexing: Running", start_time=start_time) db_dict = {i: r for i, r in enumerate(db_iter)} result_count = len(db_dict) # Sort db_dict by field values tag_dict = {k: v[field] for k, v in db_dict.items()} if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend) printMessage("Indexing: Done", start_time=start_time, end=True) # Iterate over records start_time = time() rec_count = 0 for key in sorted_keys: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Write records pass_writer.writeDict(db_dict[key]) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def sortSeqFile(seq_file, field, numeric=False, max_count=None, out_args=default_out_args): """ Sorts a sequence file by annotation fields Arguments: seq_file : filename of the sequence file to split field : position of field in sequence description to split by numeric : if True sort field numerically; if False sort field alphabetically max_count : maximum number of records in each output file if None do not create multiple files out_args : common output argument dictionary from parseCommonArgs Returns: list: output file names """ log = OrderedDict() log['START'] = 'SplitSeq' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(seq_file) log['FIELD'] = field log['NUMERIC'] = numeric log['MAX_COUNT'] = max_count printLog(log) # Open file handles in_type = getFileType(seq_file) seq_dict = readSeqFile(seq_file, index=True) if out_args['out_type'] is None: out_args['out_type'] = in_type # Get annotations and sort seq_dict by annotation values tag_dict = { k: parseAnnotation(seq_dict[k].description, delimiter=out_args['delimiter'])[field] for k in seq_dict } if numeric: tag_dict = {k: float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get) # Determine total numbers of records rec_count = len(seq_dict) if max_count >= rec_count: max_count = None # Open initial output file handles file_count = 1 if max_count is None: out_label = 'sorted' else: out_label = 'sorted-part%06i' % file_count out_handle = getOutputHandle(seq_file, out_label, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) out_files = [out_handle.name] # Loop through sorted sequence dictionary keys start_time = time() last_tag = None saved_keys = [] seq_count = chunk_count = 0 for key in sorted_keys: # Print progress for previous iteration and update count printProgress(seq_count, rec_count, 0.05, start_time=start_time) seq_count += 1 # Write saved group of sequences when tag changes if last_tag is not None and tag_dict[key] != last_tag: # Open new output file if needed if max_count is not None and chunk_count + len( saved_keys) > max_count: # Update partition counts file_count += 1 chunk_count = 0 # Open new file handle out_handle.close() out_handle = getOutputHandle(seq_file, 'sorted-part%06i' % file_count, out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Append output file name to out_files out_files.append(out_handle.name) # Write saved sequences for k in saved_keys: chunk_count += 1 SeqIO.write(seq_dict[k], out_handle, out_args['out_type']) # Reset saved keys to current key only saved_keys = [key] else: # Update list of saved keys if tag is unchanged saved_keys.append(key) # Check if total records reached, write all saved keys, and exit loop if seq_count == rec_count: for k in saved_keys: chunk_count += 1 SeqIO.write(seq_dict[k], out_handle, out_args['out_type']) out_handle.close() break # Update tag tracker last_tag = tag_dict[key] # Print log printProgress(seq_count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, f in enumerate(out_files): log['OUTPUT%i' % (i + 1)] = os.path.basename(f) log['SEQUENCES'] = seq_count log['PARTS'] = len(out_files) log['END'] = 'SplitSeq' printLog(log) # Close file handles out_handle.close() return out_files
def collectQueue(alive, result_queue, collect_queue, db_file, out_args, cluster_func=None, cluster_args={}): """ Assembles results from a queue of individual sequence results and manages log/file I/O Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues if False exit process result_queue = a multiprocessing.Queue holding processQueue results collect_queue = a multiprocessing.Queue to store collector return values db_file = the input database file name out_args = common output argument dictionary from parseCommonArgs cluster_func = the function to call for carrying out clustering on distance matrix cluster_args = a dictionary of arguments to pass to cluster_func Returns: None (adds 'log' and 'out_files' to collect_dict) """ # Open output files try: # Count records and define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] result_count = countDbFile(db_file) # Defined successful output handle pass_handle = getOutputHandle(db_file, out_label='clone-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE') # Defined failed alignment output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, out_label='clone-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None fail_writer = None # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: #sys.stderr.write('Exception in collector file opening step\n') alive.value = False raise # Get results from queue and write to files try: #print 'START COLLECT', alive.value # Iterator over results queue until sentinel object reached start_time = time() rec_count = clone_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break #print "COLLECT", alive.value, result['id'] # Print progress for previous iteration and update record count if rec_count == 0: print('PROGRESS> Assigning clones') printProgress(rec_count, result_count, 0.05, start_time) rec_count += len(result.data) # Write passed and failed records if result: for clone in result.results.values(): clone_count += 1 for i, rec in enumerate(clone): rec.annotations['CLONE'] = clone_count pass_writer.writerow(rec.toDict()) pass_count += 1 result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction) else: for i, rec in enumerate(result.data): if fail_writer is not None: fail_writer.writerow(rec.toDict()) fail_count += 1 result.log['CLONE0-%i' % (i + 1)] = str(rec.junction) # Write log printLog(result.log, handle=log_handle) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print total counts printProgress(rec_count, result_count, 0.05, start_time) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() # Update return list log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) except: #sys.stderr.write('Exception in collector result processing step\n') alive.value = False raise return None
def modifyHeaders(seq_file, modify_func, modify_args, out_args=default_out_args): """ Modifies sequence headers Arguments: seq_file = the sequence file name modify_func = the function defining the modification operation modify_args = a dictionary of arguments to pass to modify_func out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define subcommand label dictionary cmd_dict = {addHeader: 'add', copyHeader: 'copy', collapseHeader: 'collapse', deleteHeader: 'delete', expandHeader: 'expand', renameHeader: 'rename'} # Print parameter info log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = cmd_dict.get(modify_func, modify_func.__name__) log['FILE'] = os.path.basename(seq_file) for k in sorted(modify_args): v = modify_args[k] log[k.upper()] = ','.join(v) if isinstance(v, list) else v printLog(log) # Open file handles in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type out_handle = getOutputHandle(seq_file, 'reheader', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) # Count records result_count = countSeqFile(seq_file) # Iterate over sequences start_time = time() seq_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time) #Update counts seq_count += 1 # Modify header header = parseAnnotation(seq.description, delimiter=out_args['delimiter']) header = modify_func(header, delimiter=out_args['delimiter'], **modify_args) # Write new sequence seq.id = seq.name = flattenAnnotation(header, delimiter=out_args['delimiter']) seq.description = '' SeqIO.write(seq, out_handle, out_args['out_type']) # print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def tableHeaders(seq_file, fields, out_args=default_out_args): """ Builds a table of sequence header annotations Arguments: seq_file = the sequence file name fields = the list of fields to output out_args = common output argument dictionary from parseCommonArgs Returns: the output table file name """ log = OrderedDict() log['START'] = 'ParseHeaders' log['COMMAND'] = 'table' log['FILE'] = os.path.basename(seq_file) printLog(log) # Open file handles seq_iter = readSeqFile(seq_file) out_handle = getOutputHandle(seq_file, out_label='headers', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Count records result_count = countSeqFile(seq_file) # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration printProgress(seq_count, result_count, 0.05, start_time) # Get annotations seq_count += 1 ann = parseAnnotation(seq.description, fields, delimiter=out_args['delimiter']) # Write records if ann: pass_count += 1 out_writer.writerow(ann) else: fail_count += 1 # Print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseHeaders' printLog(log) # Close file handles out_handle.close() return out_handle.name
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Write germline sequences to tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'False' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader references = readRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS'] # Create output file handle and Db writer pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None fail_writer = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) pass_count = fail_count = 0 # Iterate over rows for i, row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) result_log, germlines = joinGermline(row, references, seq_field=seq_field, v_field=v_field, germ_types=germ_types) # Add germline field(s) to dictionary if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full'] if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask'] if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly'] if 'regions' in germ_types: row['GERMLINE_REGIONS'] = germlines['regions'] # Write row to pass or fail file if 'ERROR' in result_log: fail_count += 1 if fail_writer is not None: fail_writer.writerow(row) else: result_log['SEQUENCE'] = row[seq_field] result_log['GERMLINE'] = germlines['full'] result_log['REGIONS'] = germlines['regions'] pass_count += 1 pass_writer.writerow(row) printLog(result_log, handle=log_handle) # Print log printProgress(i + 1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def sortDbFile(db_file, field, numeric=False, descend=False, out_args=default_out_args): """ Sorts records by values in an annotation field Arguments: db_file = the database filename field = the field name to sort by numeric = if True sort field numerically; if False sort field alphabetically descend = if True sort in descending order; if False sort in ascending order out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'sort' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUMERIC'] = numeric printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Store all records in a dictionary start_time = time() printMessage("Indexing: Running", start_time=start_time) db_dict = {i:r for i, r in enumerate(db_iter)} result_count = len(db_dict) # Sort db_dict by field values tag_dict = {k:v[field] for k, v in db_dict.items()} if numeric: tag_dict = {k:float(v or 0) for k, v in tag_dict.items()} sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend) printMessage("Indexing: Done", start_time=start_time, end=True) # Iterate over records start_time = time() rec_count = 0 for key in sorted_keys: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write records pass_writer.writerow(db_dict[key]) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def assembleCloneGermline(db_file, repo, seq_field=default_seq_field, v_field=default_v_field, germ_types=default_germ_types, out_args=default_out_args): """ Assemble one germline sequence for each clone in a tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'True' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader references = readRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] if 'regions' in germ_types: add_fields += ['GERMLINE_REGIONS'] add_fields += ['GERMLINE_V_CALL'] add_fields += ['GERMLINE_D_CALL'] add_fields += ['GERMLINE_J_CALL'] # Create output file handle and Db writer writers = {} pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None writers['fail'] = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) counts = {} clone_count = counts['pass'] = counts['fail'] = 0 # Iterate over rows clone = 'initial' clone_dict = OrderedDict() for i, row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) # Clone isn't over yet if row.get('CLONE', '') == clone: clone_dict[i] = row # Clone just finished elif clone_dict: clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, references, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Now deal with current row (first of next clone) clone = row['CLONE'] clone_dict = OrderedDict([(i, row)]) # Last case is only for first row of file else: clone = row['CLONE'] clone_dict = OrderedDict([(i, row)]) clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, references, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Print log printProgress(i + 1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = counts['pass'] log['FAIL'] = counts['fail'] log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def convertToBaseline(db_file, id_field=default_id_field, seq_field=default_seq_field, germ_field=default_germ_field, cluster_field=None, meta_fields=None, out_file=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file : the database file name. id_field : the field containing identifiers. seq_field : the field containing sample sequences. germ_field : the field containing germline sequences. cluster_field : the field containing clonal groupings; if None write the germline for each record. meta_fields : a list of fields to add to sequence annotations. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name """ log = OrderedDict() log['START'] = 'ConvertDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field log['GERM_FIELD'] = germ_field log['CLUSTER_FIELD'] = cluster_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) result_count = countDbFile(db_file) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='clip') # Iterate over records start_time = time() rec_count, germ_count, pass_count, fail_count = 0, 0, 0, 0 cluster_last = None for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Update cluster ID cluster = rec.get(cluster_field, None) # Get germline SeqRecord when needed if cluster_field is None: germ = buildSeqRecord(rec, id_field, germ_field, meta_fields) germ.id = '>' + germ.id elif cluster != cluster_last: germ = buildSeqRecord(rec, cluster_field, germ_field) germ.id = '>' + germ.id else: germ = None # Get read SeqRecord seq = buildSeqRecord(rec, id_field, seq_field, meta_fields) # Write germline if germ is not None: germ_count += 1 SeqIO.write(germ, pass_handle, 'fasta') # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, 'fasta') else: fail_count += 1 # Set last cluster ID cluster_last = cluster # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GERMLINES'] = germ_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ConvertDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def writeDb(db_gen, file_prefix, total_count, id_dict={}, no_parse=True, score_fields=False, region_fields=False, out_args=default_out_args): """ Writes tab-delimited database file in output directory Arguments: db_gen = a generator of IgRecord objects containing alignment data file_prefix = directory and prefix for CLIP tab-delim file total_count = number of records (for progress bar) id_dict = a dictionary of {IMGT ID: full seq description} no_parse = if ID is to be parsed for pRESTO output with default delimiters score_fields = if True add alignment score fields to output file region_fields = if True add FWR and CDR region fields to output file out_args = common output argument dictionary from parseCommonArgs Returns: None """ pass_file = "%s_db-pass.tab" % file_prefix fail_file = "%s_db-fail.tab" % file_prefix ordered_fields = ['SEQUENCE_ID', 'SEQUENCE_INPUT', 'FUNCTIONAL', 'IN_FRAME', 'STOP', 'MUTATED_INVARIANT', 'INDELS', 'V_CALL', 'D_CALL', 'J_CALL', 'SEQUENCE_VDJ', 'SEQUENCE_IMGT', 'V_SEQ_START', 'V_SEQ_LENGTH', 'V_GERM_START_VDJ', 'V_GERM_LENGTH_VDJ', 'V_GERM_START_IMGT', 'V_GERM_LENGTH_IMGT', 'N1_LENGTH', 'D_SEQ_START', 'D_SEQ_LENGTH', 'D_GERM_START', 'D_GERM_LENGTH', 'N2_LENGTH', 'J_SEQ_START', 'J_SEQ_LENGTH', 'J_GERM_START', 'J_GERM_LENGTH', 'JUNCTION_LENGTH', 'JUNCTION'] if score_fields: ordered_fields.extend(['V_SCORE', 'V_IDENTITY', 'V_EVALUE', 'V_BTOP', 'J_SCORE', 'J_IDENTITY', 'J_EVALUE', 'J_BTOP']) if region_fields: ordered_fields.extend(['FWR1_IMGT', 'FWR2_IMGT', 'FWR3_IMGT', 'FWR4_IMGT', 'CDR1_IMGT', 'CDR2_IMGT', 'CDR3_IMGT']) # TODO: This is not the best approach. should pass in output fields. # Initiate passed handle pass_handle = None # Open failed file if out_args['failed']: fail_handle = open(fail_file, 'wt') fail_writer = getDbWriter(fail_handle, add_fields=['SEQUENCE_ID', 'SEQUENCE_INPUT']) else: fail_handle = None fail_writer = None # Initialize counters and file pass_writer = None start_time = time() rec_count = pass_count = fail_count = 0 for record in db_gen: #printProgress(i + (total_count/2 if id_dict else 0), total_count, 0.05, start_time) printProgress(rec_count, total_count, 0.05, start_time) rec_count += 1 # Count pass or fail if (record.v_call == 'None' and record.j_call == 'None') or \ record.functional is None or \ not record.seq_vdj or \ not record.junction: # print(record.v_call, record.j_call, record.functional, record.junction) fail_count += 1 if fail_writer is not None: fail_writer.writerow(record.toDict()) continue else: pass_count += 1 # Build sample sequence description if record.id in id_dict: record.id = id_dict[record.id] # Parse sequence description into new columns if not no_parse: record.annotations = parseAnnotation(record.id, delimiter=out_args['delimiter']) record.id = record.annotations['ID'] del record.annotations['ID'] # TODO: This is not the best approach. should pass in output fields. # If first sequence, use parsed description to create new columns and initialize writer if pass_writer is None: if not no_parse: ordered_fields.extend(list(record.annotations.keys())) pass_handle = open(pass_file, 'wt') pass_writer = getDbWriter(pass_handle, add_fields=ordered_fields) # Write row to tab-delim CLIP file pass_writer.writerow(record.toDict()) # Print log #printProgress(i+1 + (total_count/2 if id_dict else 0), total_count, 0.05, start_time) printProgress(rec_count, total_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = pass_file log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'MakeDb' printLog(log) if pass_handle is not None: pass_handle.close() if fail_handle is not None: fail_handle.close()
def convertToGenbank(db_file, inference=None, db_xref=None, molecule=default_molecule, product=default_product, features=None, c_field=None, label=None, count_field=None, index_field=None, allow_stop=False, asis_id=False, asis_calls=False, allele_delim=default_allele_delim, build_asn=False, asn_template=None, tbl2asn_exec=default_tbl2asn_exec, format=default_format, out_file=None, out_args=default_out_args): """ Builds GenBank submission fasta and table files Arguments: db_file : the database file name. inference : reference alignment tool. db_xref : reference database link. molecule : source molecule (eg, "mRNA", "genomic DNA") product : Product (protein) name. features : dictionary of sample features (BioSample attributes) to add to the description of each record. c_field : column containing the C region gene call. label : a string to use as a label for the ID. if None do not add a field label. count_field : field name to populate the AIRR_READ_COUNT note. index_field : field name to populate the AIRR_CELL_INDEX note. allow_stop : if True retain records with junctions having stop codons. asis_id : if True use the original sequence ID for the output IDs. asis_calls : if True do not parse gene calls for IMGT nomenclature. allele_delim : delimiter separating the gene name from the allele number when asis_calls=True. build_asn : if True run tbl2asn on the generated .tbl and .fsa files. asn_template : template file (.sbt) to pass to tbl2asn. tbl2asn_exec : name of or path to the tbl2asn executable. format : input and output format. out_file : output file name without extension. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: tuple : the output (feature table, fasta) file names. """ log = OrderedDict() log['START'] = 'ConvertDb' log['COMMAND'] = 'genbank' log['FILE'] = os.path.basename(db_file) printLog(log) # Define format operators try: reader, __, schema = getFormatOperators(format) except ValueError: printError('Invalid format %s.' % format) # Open input db_handle = open(db_file, 'rt') db_iter = reader(db_handle) # Check for required columns try: required = ['sequence_input', 'v_call', 'd_call', 'j_call', 'v_seq_start', 'd_seq_start', 'j_seq_start'] checkFields(required, db_iter.fields, schema=schema) except LookupError as e: printError(e) # Open output if out_file is not None: out_name, __ = os.path.splitext(out_file) fsa_handle = open('%s.fsa' % out_name, 'w') tbl_handle = open('%s.tbl' % out_name, 'w') else: fsa_handle = getOutputHandle(db_file, out_label='genbank', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='fsa') tbl_handle = getOutputHandle(db_file, out_label='genbank', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tbl') # Count records result_count = countDbFile(db_file) # Define writer writer = csv.writer(tbl_handle, delimiter='\t', quoting=csv.QUOTE_NONE) # Iterate over records start_time = time() rec_count, pass_count, fail_count = 0, 0, 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Extract table dictionary name = None if asis_id else rec_count seq = makeGenbankSequence(rec, name=name, label=label, count_field=count_field, index_field=index_field, molecule=molecule, features=features) tbl = makeGenbankFeatures(rec, start=seq['start'], end=seq['end'], product=product, db_xref=db_xref, inference=inference, c_field=c_field, allow_stop=allow_stop, asis_calls=asis_calls, allele_delim=allele_delim) if tbl is not None: pass_count +=1 # Write table writer.writerow(['>Features', seq['record'].id]) for feature, qualifiers in tbl.items(): writer.writerow(feature) if qualifiers: for x in qualifiers: writer.writerow(list(chain(['', '', ''], x))) # Write sequence SeqIO.write(seq['record'], fsa_handle, 'fasta') else: fail_count += 1 # Final progress bar printProgress(rec_count, result_count, 0.05, start_time=start_time) # Run tbl2asn if build_asn: start_time = time() printMessage('Running tbl2asn', start_time=start_time, width=25) result = runASN(fsa_handle.name, template=asn_template, exec=tbl2asn_exec) printMessage('Done', start_time=start_time, end=True, width=25) # Print ending console log log = OrderedDict() log['OUTPUT_TBL'] = os.path.basename(tbl_handle.name) log['OUTPUT_FSA'] = os.path.basename(fsa_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ConvertDb' printLog(log) # Close file handles tbl_handle.close() fsa_handle.close() db_handle.close() return (tbl_handle.name, fsa_handle.name)
def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sequences meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles out_type = 'fasta' db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Get SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter']) # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, out_type) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def indexDbFile(db_file, field=default_index_field, out_file=None, out_args=default_out_args): """ Adds an index column to a database file Arguments: db_file : the database file name. field : the name of the index field to add. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'index' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) __, __, out_args['out_type'] = splitName(db_file) # Append index field out_fields = list(db_iter.fields) out_fields.append(field) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Add count and write updated row rec.update({field: rec_count}) pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def addDbFile(db_file, fields, values, out_args=default_out_args): """ Adds field and value pairs to a database file Arguments: db_file = the database file name fields = a list of fields to add values = a list of values to assign to all rows of each field out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'add' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields) # Count records result_count = countDbFile(db_file) # Define fields and values to append add_dict = { k: v for k, v in zip(fields, values) if k not in db_iter.fieldnames } # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Write updated row rec.update(add_dict) pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args): """ Divides a tab-delimited database file into segments by description tags Arguments: db_file : filename of the tab-delimited database file to split field : the field name by which to split db_file num_split : the numerical threshold by which to group sequences; if None treat field as textual out_args : common output argument dictionary from parseCommonArgs Returns: list : a list of output file names. """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'split' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUM_SPLIT'] = num_split printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) out_fields = db_iter.fields __, __, out_args['out_type'] = splitName(db_file) # Determine total numbers of records rec_count = countDbFile(db_file) start_time = time() count = 0 # Sort records into files based on textual field if num_split is None: # Create set of unique field tags with open(db_file, 'rt') as tmp_handle: tmp_iter = TSVReader(tmp_handle) tag_list = list(set([row[field] for row in tmp_iter])) # Forbidden characters in filename and replacements no_good = { '\/': 'f', '\\': 'b', '?': 'q', '\%': 'p', '*': 's', ':': 'c', '\|': 'pi', '\"': 'dq', '\'': 'sq', '<': 'gt', '>': 'lt', ' ': '_' } # Replace forbidden characters in tag_list tag_dict = {} for tag in tag_list: for c, r in no_good.items(): tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \ if c in tag else tag_dict.get(tag, tag)) # Create output handles handles_dict = { tag: getOutputHandle(db_file, out_label='%s-%s' % (field, label), out_name=out_args['out_name'], out_dir=out_args['out_dir'], out_type=out_args['out_type']) for tag, label in tag_dict.items() } # Create Db writer instances writers_dict = { tag: TSVWriter(handles_dict[tag], fields=out_fields) for tag in tag_dict } # Iterate over records for row in db_iter: printProgress(count, rec_count, 0.05, start_time=start_time) count += 1 # Write row to appropriate file tag = row[field] writers_dict[tag].writeDict(row) # Sort records into files based on numeric num_split else: num_split = float(num_split) # Create output handles handles_dict = { 'under': getOutputHandle(db_file, out_label='under-%.1f' % num_split, out_name=out_args['out_name'], out_dir=out_args['out_dir'], out_type=out_args['out_type']), 'atleast': getOutputHandle(db_file, out_label='atleast-%.1f' % num_split, out_name=out_args['out_name'], out_dir=out_args['out_dir'], out_type=out_args['out_type']) } # Create Db writer instances writers_dict = { 'under': TSVWriter(handles_dict['under'], fields=out_fields), 'atleast': TSVWriter(handles_dict['atleast'], fields=out_fields) } # Iterate over records for row in db_iter: printProgress(count, rec_count, 0.05, start_time=start_time) count += 1 tag = row[field] tag = 'under' if float(tag) < num_split else 'atleast' writers_dict[tag].writeDict(row) # Write log printProgress(count, rec_count, 0.05, start_time=start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['RECORDS'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'ParseDb' printLog(log) # Close output file handles db_handle.close() for t in handles_dict: handles_dict[t].close() return [handles_dict[t].name for t in handles_dict]
def assembleEachGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Write germline sequences to tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'False' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader repo_dict = getRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] # Create output file handle and Db writer pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) fail_writer = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None fail_writer = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) pass_count = fail_count = 0 # Iterate over rows for i,row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) result_log, germlines = joinGermline(row, repo_dict, germ_types, v_field, seq_field) # Add germline field(s) to dictionary if 'full' in germ_types: row['GERMLINE_' + seq_type] = germlines['full'] if 'dmask' in germ_types: row['GERMLINE_' + seq_type + '_D_MASK'] = germlines['dmask'] if 'vonly' in germ_types: row['GERMLINE_' + seq_type + '_V_REGION'] = germlines['vonly'] # Write row to pass or fail file if 'ERROR' in result_log: fail_count += 1 if fail_writer is not None: fail_writer.writerow(row) else: result_log['SEQUENCE'] = row[seq_field] result_log['GERMLINE'] = germlines['full'] result_log['REGIONS'] = germlines['regions'] pass_count += 1 pass_writer.writerow(row) printLog(result_log, handle=log_handle) # Print log printProgress(i+1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def selectDbFile(db_file, fields, values, logic='any', regex=False, out_file=None, out_args=default_out_args): """ Selects records from a database file Arguments: db_file : the database file name fields : a list of fields to check for selection criteria values : a list of values defining selection targets logic : one of 'any' or 'all' defining whether one or all fields must have a match. regex : if False do exact full string matches; if True allow partial regex matches. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs Returns: str : output file name. """ # Define string match function if regex: def _match_func(x, patterns): return any([re.search(p, x) for p in patterns]) else: def _match_func(x, patterns): return x in patterns # Define logic function if logic == 'any': _logic_func = any elif logic == 'all': _logic_func = all # Print console log log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'select' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) log['REGEX'] = regex printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) out_fields = db_iter.fields __, __, out_args['out_type'] = splitName(db_file) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count, pass_count, fail_count = 0, 0, 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Check for selection values in all fields select = _logic_func( [_match_func(rec.get(f, False), values) for f in fields]) # Write sequences if select: pass_count += 1 pass_writer.writeDict(rec) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['SELECTED'] = pass_count log['DISCARDED'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def indexJunctions(db_iter, fields=None, mode='gene', action='first'): """ Identifies preclonal groups by V, J and junction length Arguments: db_iter = an iterator of IgRecords defined by readDbFile fields = additional annotation fields to use to group preclones; if None use only V, J and junction length mode = specificity of alignment call to use for assigning preclones; one of ('allele', 'gene') action = how to handle multiple value fields when assigning preclones; one of ('first', 'set') Returns: a dictionary of {(V, J, junction length):[IgRecords]} """ # Define functions for grouping keys if mode == 'allele' and fields is None: def _get_key(rec, act): return (rec.getVAllele(act), rec.getJAllele(act), None if rec.junction is None else len(rec.junction)) elif mode == 'gene' and fields is None: def _get_key(rec, act): return (rec.getVGene(act), rec.getJGene(act), None if rec.junction is None else len(rec.junction)) elif mode == 'allele' and fields is not None: def _get_key(rec, act): vdj = [rec.getVAllele(act), rec.getJAllele(act), None if rec.junction is None else len(rec.junction)] ann = [rec.toDict().get(k, None) for k in fields] return tuple(chain(vdj, ann)) elif mode == 'gene' and fields is not None: def _get_key(rec, act): vdj = [rec.getVGene(act), rec.getJGene(act), None if rec.junction is None else len(rec.junction)] ann = [rec.toDict().get(k, None) for k in fields] return tuple(chain(vdj, ann)) start_time = time() clone_index = {} rec_count = 0 for rec in db_iter: key = _get_key(rec, action) # Print progress if rec_count == 0: print('PROGRESS> Grouping sequences') printProgress(rec_count, step=1000, start_time=start_time) rec_count += 1 # Assigned passed preclone records to key and failed to index None if all([k is not None and k != '' for k in key]): #print key # TODO: Has much slow. Should have less slow. if action == 'set': f_range = list(range(2, 3 + (len(fields) if fields else 0))) vdj_range = list(range(2)) # Check for any keys that have matching columns and junction length and overlapping genes/alleles to_remove = [] if len(clone_index) > (1 if None in clone_index else 0) and key not in clone_index: key = list(key) for k in clone_index: if k is not None and all([key[i] == k[i] for i in f_range]): if all([not set(key[i]).isdisjoint(set(k[i])) for i in vdj_range]): for i in vdj_range: key[i] = tuple(set(key[i]).union(set(k[i]))) to_remove.append(k) # Remove original keys, replace with union of all genes/alleles and append values to new key val = [rec] val += list(chain(*(clone_index.pop(k) for k in to_remove))) clone_index[tuple(key)] = clone_index.get(tuple(key),[]) + val elif action == 'first': clone_index.setdefault(key, []).append(rec) else: clone_index.setdefault(None, []).append(rec) printProgress(rec_count, step=1000, start_time=start_time, end=True) return clone_index
def updateDbFile(db_file, field, values, updates, out_file=None, out_args=default_out_args): """ Updates field and value pairs to a database file Arguments: db_file : the database file name. field : the field to update. values : a list of values to specifying which rows to update. updates : a list of values to update each value with. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'update' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['VALUES'] = ','.join(values) log['UPDATES'] = ','.join(updates) printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = TSVReader(db_handle) out_fields = db_iter.fields __, __, out_args['out_type'] = splitName(db_file) # Open output if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) pass_writer = TSVWriter(pass_handle, out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count, pass_count = 0, 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Updated values if found for x, y in zip(values, updates): if rec[field] == x: rec[field] = y pass_count += 1 # Write records pass_writer.writeDict(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['UPDATED'] = pass_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def convertDbClip(db_file, id_field=default_id_field, seq_field=default_seq_field, germ_field=default_germ_field, cluster_field=None, meta_fields=None, out_args=default_out_args): """ Builds fasta files from database records Arguments: db_file = the database file name id_field = the field containing identifiers seq_field = the field containing sample sequences germ_field = the field containing germline sequences cluster_field = the field containing clonal groupings if None write the germline for each record meta_fields = a list of fields to add to sequence annotations out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'fasta' log['FILE'] = os.path.basename(db_file) log['ID_FIELD'] = id_field log['SEQ_FIELD'] = seq_field log['GERM_FIELD'] = germ_field log['CLUSTER_FIELD'] = cluster_field if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='clip') # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = germ_count = pass_count = fail_count = 0 cluster_last = None for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Update cluster ID cluster = rec.get(cluster_field, None) # Get germline SeqRecord when needed if cluster_field is None: germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields, delimiter=out_args['delimiter']) germ.id = '>' + germ.id elif cluster != cluster_last: germ = getDbSeqRecord(rec, cluster_field, germ_field, delimiter=out_args['delimiter']) germ.id = '>' + germ.id else: germ = None # Get read SeqRecord seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, delimiter=out_args['delimiter']) # Write germline if germ is not None: germ_count += 1 SeqIO.write(germ, pass_handle, 'fasta') # Write sequences if seq is not None: pass_count += 1 SeqIO.write(seq, pass_handle, 'fasta') else: fail_count += 1 # Set last cluster ID cluster_last = cluster # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['GERMLINES'] = germ_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def tableLog(record_file, fields, out_args=default_out_args): """ Converts a pRESTO log to a table of annotations Arguments: record_file = the log file name fields = the list of fields to output out_args = common output argument dictionary from parseCommonArgs Returns: the output table file name """ log = OrderedDict() log['START'] = 'ParseLog' log['FILE'] = os.path.basename(record_file) printLog(log) # Open file handles log_handle = open(record_file, 'rU') out_handle = getOutputHandle(record_file, 'table', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Open csv writer and write header out_writer = csv.DictWriter(out_handle, extrasaction='ignore', restval='', delimiter='\t', fieldnames=fields) out_writer.writeheader() # Iterate over log records start_time = time() record = '' rec_count = pass_count = fail_count = 0 for line in log_handle: if line.strip() == '' and record: # Print progress for previous iteration printProgress(rec_count, None, 1e5, start_time) # Parse record block rec_count += 1 record_dict = parseLogRecord(record) # Write records if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Empty record string record = '' else: # Append to record record += line else: # Write final record if record: record_dict = parseLogRecord(record) if any([f in fields for f in record_dict]): pass_count += 1 out_writer.writerow(record_dict) elif record_dict: fail_count += 1 # Print counts printProgress(rec_count, None, 1e5, start_time, end=True) log = OrderedDict() log['OUTPUT'] = os.path.basename(out_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ParseLog' printLog(log) # Close file handles log_handle.close() out_handle.close() return log_handle.name
def findUniqueSeq(uniq_dict, search_keys, seq_dict, max_missing=default_max_missing, uniq_fields=None, copy_fields=None, max_field=None, min_field=None, inner=False, delimiter=default_delimiter): """ Finds unique sequences Arguments: uniq_dict = a dictionary of unique sequences generated by findUniqueSeq() search_keys = a list containing the subset of dictionary keys to be checked seq_dict = a SeqRecords dictionary generated by SeqIO.index() max_missing = the number of missing characters to allow in a unique sequences uniq_fields = a list of annotations that define a sequence as unique if they differ copy_fields = a list of annotations to copy into unique sequence annotations max_field = a numeric field whose maximum value determines the retained sequence min_field = a numeric field whose minimum value determines the retained sequence inner = if True exclude consecutive outer ambiguous characters from iterations and matching delimiter = description field delimiter Returns: a tuple of (uniq_dict, search_keys, dup_keys) modified from passed values """ # Define local variables ambig_re = re.compile(r'[\.\-N]') score = (max_missing > 0) dup_keys = [] to_remove = [] start_time = time() result_count = len(search_keys) print('MISSING> %i' % max_missing) # Iterate over search keys and update uniq_dict and dup_keys for idx, key in enumerate(search_keys): # Print progress of previous iteration printProgress(idx, result_count, 0.05, start_time) # Define sequence to process seq = seq_dict[key] seq_str = str(seq.seq) if inner: seq_str = seq_str.strip('.-N') # Skip processing of ambiguous sequences over max_missing threshold ambig_count = len(ambig_re.findall(seq_str)) if ambig_count > max_missing: continue # Parse annotation and define unique identifiers (uid) if uniq_fields is not None: ann = parseAnnotation(seq_dict[key].description, uniq_fields, delimiter=delimiter) uid = tuple(chain([seq_str], list(ann.values()))) else: uid = (seq_str, None) # Parse annotation and define copied identifiers (cid) if copy_fields is not None: ann = parseAnnotation(seq.description, copy_fields, delimiter=delimiter) #print ann #cid = [[a] for a in ann.values()] cid = [[ann.get(k)] for k in copy_fields] #print cid else: cid = [] # Store new unique sequences and process duplicates match = findUID(uid, uniq_dict, score) if match is None: uniq_dict[uid] = list(chain([seq, 1, ambig_count], cid)) else: # Updated sequence, count, ambiguous character count, and count sets dup_key = key uniq_dict[match][1] += 1 for x, c in enumerate(cid): uniq_dict[match][3 + x].extend(c) # Check whether to replace previous unique sequence with current sequence if ambig_count <= uniq_dict[match][2]: swap = False seq_last = uniq_dict[match][0] if max_field is not None: swap = float(parseAnnotation(seq.description, delimiter=delimiter)[max_field]) > \ float(parseAnnotation(seq_last.description, delimiter=delimiter)[max_field]) elif min_field is not None: swap = float(parseAnnotation(seq.description, delimiter=delimiter)[min_field]) > \ float(parseAnnotation(seq_last.description, delimiter=delimiter)[min_field]) # TODO: quality evaluation is a bottleneck else: if hasattr(seq, 'letter_annotations') and 'phred_quality' in seq.letter_annotations: q_this = float(sum(seq.letter_annotations['phred_quality'])) / len(seq) q_last = float(sum(seq_last.letter_annotations['phred_quality'])) / len(seq_last) swap = q_this > q_last # Replace old sequence if criteria passed if swap: dup_key = seq_last.id #uniq_dict[match] = [seq, uniq_dict[match][1], ambig_count] uniq_dict[match][0] = seq uniq_dict[match][2] = ambig_count # Update duplicate list dup_keys.append(dup_key) # Mark seq for removal from later steps to_remove.append(idx) # Remove matched sequences from search_keys for j in reversed(to_remove): del search_keys[j] # Update progress printProgress(result_count, result_count, 0.05, start_time) return (uniq_dict, search_keys, dup_keys)
def collectQueueClust(alive, result_queue, collect_queue, db_file, out_args, cluster_func, cluster_args): """ Assembles results from a queue of individual sequence results and manages log/file I/O Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues if False exit process result_queue = a multiprocessing.Queue holding processQueue results collect_queue = a multiprocessing.Queue to store collector return values db_file = the input database file name out_args = common output argument dictionary from parseCommonArgs cluster_func = the function to call for carrying out clustering on distance matrix cluster_args = a dictionary of arguments to pass to cluster_func Returns: None (adds 'log' and 'out_files' to collect_dict) """ # Open output files try: # Iterate over Ig records to count and order by junction length result_count = 0 records = {} # print 'Reading file...' db_iter = readDbFile(db_file) for rec in db_iter: records[rec.id] = rec result_count += 1 records = OrderedDict(sorted(list(records.items()), key=lambda i: i[1].junction_length)) # Define empty matrix to store assembled results dist_mat = np.zeros((result_count,result_count)) # Count records and define output format out_type = getFileType(db_file) if out_args['out_type'] is None \ else out_args['out_type'] # Defined successful output handle pass_handle = getOutputHandle(db_file, out_label='clone-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) pass_writer = getDbWriter(pass_handle, db_file, add_fields='CLONE') # Defined failed cloning output handle if out_args['failed']: fail_handle = getOutputHandle(db_file, out_label='clone-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_type) fail_writer = getDbWriter(fail_handle, db_file) else: fail_handle = None fail_writer = None # Open log file if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() row_count = rec_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration if row_count == 0: print('PROGRESS> Assigning clones') printProgress(row_count, result_count, 0.05, start_time) # Update counts for iteration row_count += 1 rec_count += len(result) # Add result row to distance matrix if result: dist_mat[list(range(result_count-len(result),result_count)),result_count-len(result)] = result.results else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Calculate linkage and carry out clustering # print dist_mat clusters = cluster_func(dist_mat, **cluster_args) if dist_mat is not None else None clones = {} # print clusters for i, c in enumerate(clusters): clones.setdefault(c, []).append(records[list(records.keys())[i]]) # Write passed and failed records clone_count = pass_count = fail_count = 0 if clones: for clone in clones.values(): clone_count += 1 for i, rec in enumerate(clone): rec.annotations['CLONE'] = clone_count pass_writer.writerow(rec.toDict()) pass_count += 1 #result.log['CLONE%i-%i' % (clone_count, i + 1)] = str(rec.junction) else: for i, rec in enumerate(result.data): fail_writer.writerow(rec.toDict()) fail_count += 1 #result.log['CLONE0-%i' % (i + 1)] = str(rec.junction) # Print final progress printProgress(row_count, result_count, 0.05, start_time) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close() # Update return list log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = fail_count collect_dict = {'log':log, 'out_files': [pass_handle.name]} collect_queue.put(collect_dict) except: alive.value = False raise return None
def assembleCloneGermline(db_file, repo, germ_types, v_field, seq_field, out_args=default_out_args): """ Assemble one germline sequence for each clone in a tab-delimited database file Arguments: db_file = input tab-delimited database file repo = folder with germline repertoire files germ_types = types of germline sequences to be output (full germline, D-region masked, only V-region germline) v_field = field in which to look for V call seq_field = field in which to look for sequence out_args = arguments for output preferences Returns: None """ # Print parameter info log = OrderedDict() log['START'] = 'CreateGermlines' log['DB_FILE'] = os.path.basename(db_file) log['GERM_TYPES'] = germ_types if isinstance(germ_types, str) else ','.join(germ_types) log['CLONED'] = 'True' log['V_FIELD'] = v_field log['SEQ_FIELD'] = seq_field printLog(log) # Get repertoire and open Db reader repo_dict = getRepo(repo) reader = readDbFile(db_file, ig=False) # Exit if V call field does not exist in reader if v_field not in reader.fieldnames: sys.exit('Error: V field does not exist in input database file.') # Define log handle if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') add_fields = [] seq_type = seq_field.split('_')[-1] if 'full' in germ_types: add_fields += ['GERMLINE_' + seq_type] if 'dmask' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_D_MASK'] if 'vonly' in germ_types: add_fields += ['GERMLINE_' + seq_type + '_V_REGION'] # Create output file handle and Db writer writers = {} pass_handle = getOutputHandle(db_file, 'germ-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['pass'] = getDbWriter(pass_handle, db_file, add_fields=add_fields) if out_args['failed']: fail_handle = getOutputHandle(db_file, 'germ-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) writers['fail'] = getDbWriter(fail_handle, db_file, add_fields=add_fields) else: fail_handle = None writers['fail'] = None # Initialize time and total count for progress bar start_time = time() rec_count = countDbFile(db_file) counts = {} clone_count = counts['pass'] = counts['fail'] = 0 # Iterate over rows clone = 'initial' clone_dict = OrderedDict() for i,row in enumerate(reader): # Print progress printProgress(i, rec_count, 0.05, start_time) # Clone isn't over yet if row.get('CLONE','') == clone: clone_dict[row["SEQUENCE_ID"]] = row # Clone just finished elif clone_dict: clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Now deal with current row (first of next clone) clone = row['CLONE'] clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)]) # Last case is only for first row of file else: clone = row['CLONE'] clone_dict = OrderedDict([(row['SEQUENCE_ID'],row)]) clone_count += 1 result_log = makeCloneGermline(clone, clone_dict, repo_dict, germ_types, v_field, seq_field, counts, writers, out_args) printLog(result_log, handle=log_handle) # Print log printProgress(i+1, rec_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['CLONES'] = clone_count log['RECORDS'] = rec_count log['PASS'] = counts['pass'] log['FAIL'] = counts['fail'] log['END'] = 'CreateGermlines' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() if log_handle is not None: log_handle.close()
def convertHeaders(seq_file, convert_func, convert_args={}, out_args=default_out_args): """ Converts sequence headers to the pRESTO format Arguments: seq_file = the sequence file name convert_func = the function used to convert sequence headers convert_args = a dictionary of arguments to pass to convert_func out_args = common output argument dictionary from parseCommonArgs Returns: the output sequence file name """ # Define subcommand label dictionary cmd_dict = {convertGenericHeader:'generic', convert454Header:'454', convertGenbankHeader:'genbank', convertIlluminaHeader:'illumina', convertIMGTHeader:'imgt', convertSRAHeader:'sra'} log = OrderedDict() log['START'] = 'ConvertHeaders' log['COMMAND'] = cmd_dict[convert_func] log['FILE'] = os.path.basename(seq_file) printLog(log) # Open input file in_type = getFileType(seq_file) seq_iter = readSeqFile(seq_file) if out_args['out_type'] is None: out_args['out_type'] = in_type # Count records result_count = countSeqFile(seq_file) # Open output file handles pass_handle = getOutputHandle(seq_file, 'convert-pass', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) if out_args['failed']: fail_handle = getOutputHandle(seq_file, 'convert-fail', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=out_args['out_type']) else: fail_handle = None # Set additional conversion arguments if convert_func in [convertGenericHeader, convertGenbankHeader]: convert_args.update({'delimiter':out_args['delimiter']}) # Iterate over sequences start_time = time() seq_count = pass_count = fail_count = 0 for seq in seq_iter: # Print progress for previous iteration and update count printProgress(seq_count, result_count, 0.05, start_time) seq_count += 1 # Convert header header = convert_func(seq.description, **convert_args) if header is not None: # Write successfully converted sequences pass_count += 1 seq.id = seq.name = flattenAnnotation(header, out_args['delimiter']) seq.description = '' SeqIO.write(seq, pass_handle, out_args['out_type']) else: fail_count += 1 if fail_handle is not None: # Write successfully unconverted sequences SeqIO.write(seq, fail_handle, out_args['out_type']) # Print counts printProgress(seq_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['END'] = 'ConvertHeaders' printLog(log) # Close file handles pass_handle.close() if fail_handle is not None: fail_handle.close() return pass_handle.name
def deleteDbFile(db_file, fields, values, logic='any', regex=False, out_args=default_out_args): """ Deletes records from a database file Arguments: db_file = the database file name fields = a list of fields to check for deletion criteria values = a list of values defining deletion targets logic = one of 'any' or 'all' defining whether one or all fields must have a match. regex = if False do exact full string matches; if True allow partial regex matches. out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ # Define string match function if regex: def _match_func(x, patterns): return any([re.search(p, x) for p in patterns]) else: def _match_func(x, patterns): return x in patterns # Define logic function if logic == 'any': _logic_func = any elif logic == 'all': _logic_func = all log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'delete' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['VALUES'] = ','.join(values) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-delete', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = fail_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Check for deletion values in all fields delete = _logic_func( [_match_func(rec.get(f, False), values) for f in fields]) # Write sequences if not delete: pass_count += 1 pass_writer.writerow(rec) else: fail_count += 1 # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['KEPT'] = pass_count log['DELETED'] = fail_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def renameDbFile(db_file, fields, names, out_args=default_out_args): """ Renames fields in a database file Arguments: db_file = the database file name fields = a list of fields to rename values = a list of new names for fields out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'rename' log['FILE'] = os.path.basename(db_file) log['FIELDS'] = ','.join(fields) log['NAMES'] = ','.join(names) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') # Get header and rename fields header = (readDbFile(db_file, ig=False)).fieldnames for f, n in zip(fields, names): i = header.index(f) header[i] = n # Open writer and write new header # TODO: should modify getDbWriter to take a list of fields pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab') pass_writer.writeheader() # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore # Rename fields for f, n in zip(fields, names): rec[n] = rec.pop(f) # Write pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def updateDbFile(db_file, field, values, updates, out_args=default_out_args): """ Updates field and value pairs to a database file Arguments: db_file = the database file name field = the field to update values = a list of values to specifying which rows to update updates = a list of values to update each value with out_args = common output argument dictionary from parseCommonArgs Returns: the output file name """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'update' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['VALUES'] = ','.join(values) log['UPDATES'] = ','.join(updates) printLog(log) # Open file handles db_iter = readDbFile(db_file, ig=False) pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type='tab') pass_writer = getDbWriter(pass_handle, db_file) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time) rec_count += 1 # Updated values if found for x, y in zip(values, updates): if rec[field] == x: rec[field] = y pass_count += 1 # Write records pass_writer.writerow(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['UPDATED'] = pass_count log['END'] = 'ParseDb' printLog(log) # Close file handles pass_handle.close() return pass_handle.name
def pairSeq(seq_file_1, seq_file_2, fields_1=None, fields_2=None, coord_type=default_coord_type, out_args=default_out_args): """ Generates consensus sequences Arguments: seq_file_1 = the file containing the grouped sequences and annotations seq_file_2 = the file to assign annotations to from seq_file_1 fields_1 = list of annotations in seq_file_1 records to copy to seq_file_2 records; if None do not copy any annotations fields_2 = list of annotations in seq_file_2 records to copy to seq_file_1 records; if None do not copy any annotations coord_type = the sequence header format out_args = common output argument dictionary from parseCommonArgs Returns: a list of tuples holding successfully paired filenames for (seq_file_1, seq_file_2) """ # Define private functions def _key_func(x): return getCoordKey(x, coord_type=coord_type, delimiter=out_args['delimiter']) log = OrderedDict() log['START'] = 'PairSeq' log['FILE1'] = os.path.basename(seq_file_1) log['FILE2'] = os.path.basename(seq_file_2) log['FIELDS_1'] = ','.join(fields_1) if fields_1 is not None else None log['FIELDS_2'] = ','.join(fields_2) if fields_2 is not None else None log['COORD_TYPE'] = coord_type printLog(log) # Define output type if out_args['out_type'] is None: out_type_1 = getFileType(seq_file_1) out_type_2 = getFileType(seq_file_2) else: out_type_1 = out_type_2 = out_args['out_type'] # Define output name if out_args['out_name'] is None: out_name_1 = out_name_2 = None else: out_name_1 = '%s-1' % out_args['out_name'] out_name_2 = '%s-2' % out_args['out_name'] # Open and count files start_time = time() printMessage("Indexing files", start_time=start_time) # Index file 1 seq_count_1 = countSeqFile(seq_file_1) seq_dict_1 = readSeqFile(seq_file_1, index=True, key_func=_key_func) # Define file 2 iterator seq_count_2 = countSeqFile(seq_file_2) seq_iter_2 = readSeqFile(seq_file_2, index=False) printMessage("Done", start_time=start_time, end=True) # Open output file handles pass_handle_1 = getOutputHandle(seq_file_1, 'pair-pass', out_args['out_dir'], out_name=out_name_1, out_type=out_type_1) pass_handle_2 = getOutputHandle(seq_file_2, 'pair-pass', out_args['out_dir'], out_name=out_name_2, out_type=out_type_2) if out_args['failed']: fail_handle_1 = getOutputHandle(seq_file_1, 'pair-fail', out_dir=out_args['out_dir'], out_name=out_name_1, out_type=out_type_1) fail_handle_2 = getOutputHandle(seq_file_2, 'pair-fail', out_dir=out_args['out_dir'], out_name=out_name_2, out_type=out_type_2) pass_keys = list() # Iterate over pairs and write to output files start_time = time() rec_count = pair_count = 0 for seq_2 in seq_iter_2: # Print progress for previous iteration printProgress(rec_count, seq_count_2, 0.05, start_time) rec_count += 1 # Check for file 2 mate pair in file 1 coord_2 = getCoordKey(seq_2.id, coord_type=coord_type, delimiter=out_args['delimiter']) seq_1 = seq_dict_1.get(coord_2, None) if seq_1 is not None: # Record paired keys pair_count += 1 if fields_1 is not None or fields_2 is not None: ann_1 = parseAnnotation(seq_1.description, delimiter=out_args['delimiter']) ann_2 = parseAnnotation(seq_2.description, delimiter=out_args['delimiter']) # Prepend annotations from seq_1 to seq_2 if fields_1 is not None: copy_ann = OrderedDict([(k, v) for k, v in ann_1.items() \ if k in fields_1]) merge_ann = mergeAnnotation(ann_2, copy_ann, prepend=True, delimiter=out_args['delimiter']) seq_2.id = flattenAnnotation(merge_ann, delimiter=out_args['delimiter']) seq_2.description = '' # Append annotations from seq_2 to seq_1 if fields_2 is not None: copy_ann = OrderedDict([(k, v) for k, v in ann_2.items() \ if k in fields_2]) merge_ann = mergeAnnotation(ann_1, copy_ann, prepend=False, delimiter=out_args['delimiter']) seq_1.id = flattenAnnotation(merge_ann, delimiter=out_args['delimiter']) seq_1.description = '' # Write paired records SeqIO.write(seq_1, pass_handle_1, out_type_1) SeqIO.write(seq_2, pass_handle_2, out_type_2) # Write unpaired file 2 records and updated paired key list for finding unpaired file 1 records if out_args['failed']: if seq_1 is not None: pass_keys.append(coord_2) else: SeqIO.write(seq_2, fail_handle_2, out_type_2) # Print final progress printProgress(rec_count, seq_count_2, 0.05, start_time) # Find and write unpaired file 1 records if out_args['failed']: start_time = time() printMessage("Finding unpaired", start_time=start_time) # Find file 1 unpaired keys pass_keys = set(pass_keys) unpaired = set(seq_dict_1).difference(pass_keys) # Write unpaired file 1 records for k in unpaired: SeqIO.write(seq_dict_1[k], fail_handle_1, out_type_1) printMessage("Done", start_time=start_time, end=True) # Print log log = OrderedDict() log['OUTPUT1'] = os.path.basename(pass_handle_1.name) log['OUTPUT2'] = os.path.basename(pass_handle_2.name) log['SEQUENCES1'] = seq_count_1 log['SEQUENCES2'] = seq_count_2 log['PASS'] = pair_count log['END'] = 'PairSeq' printLog(log) # Close file handles pass_handle_1.close() pass_handle_2.close() return [(pass_handle_1.name, pass_handle_2.name)]
def insertGaps(db_file, references=None, format=default_format, out_file=None, out_args=default_out_args): """ Inserts IMGT numbering into V fields Arguments: db_file : the database file name. references : folder with germline repertoire files. If None, do not updated alignment columns wtih IMGT gaps. format : input format. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name """ log = OrderedDict() log['START'] = 'ConvertDb' log['COMMAND'] = 'imgt' log['FILE'] = os.path.basename(db_file) printLog(log) # Define format operators try: reader, writer, schema = getFormatOperators(format) except ValueError: printError('Invalid format %s.' % format) # Open input db_handle = open(db_file, 'rt') db_iter = reader(db_handle) # Check for required columns try: required = ['sequence_imgt', 'v_germ_start_imgt'] checkFields(required, db_iter.fields, schema=schema) except LookupError as e: printError(e) # Load references reference_dict = readGermlines(references) # Check for IMGT-gaps in germlines if all('...' not in x for x in reference_dict.values()): printWarning('Germline reference sequences do not appear to contain IMGT-numbering spacers. Results may be incorrect.') # Open output writer if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='gap', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=schema.out_type) pass_writer = writer(pass_handle, fields=db_iter.fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = pass_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Update IMGT fields imgt_dict = correctIMGTFields(rec, reference_dict) # Write records if imgt_dict is not None: pass_count += 1 rec.setDict(imgt_dict, parse=False) pass_writer.writeReceptor(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['PASS'] = pass_count log['FAIL'] = rec_count - pass_count log['END'] = 'ConvertDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def splitDbFile(db_file, field, num_split=None, out_args=default_out_args): """ Divides a tab-delimited database file into segments by description tags Arguments: db_file = filename of the tab-delimited database file to split field = the field name by which to split db_file num_split = the numerical threshold by which to group sequences; if None treat field as textual out_args = common output argument dictionary from parseCommonArgs Returns: a list of output file names """ log = OrderedDict() log['START'] = 'ParseDb' log['COMMAND'] = 'split' log['FILE'] = os.path.basename(db_file) log['FIELD'] = field log['NUM_SPLIT'] = num_split printLog(log) # Open IgRecord reader iter object reader = readDbFile(db_file, ig=False) # Determine total numbers of records rec_count = countDbFile(db_file) start_time = time() count = 0 # Sort records into files based on textual field if num_split is None: # Create set of unique field tags tmp_iter = readDbFile(db_file, ig=False) tag_list = list(set([row[field] for row in tmp_iter])) # Forbidden characters in filename and replacements noGood = {'\/':'f','\\':'b','?':'q','\%':'p','*':'s',':':'c', '\|':'pi','\"':'dq','\'':'sq','<':'gt','>':'lt',' ':'_'} # Replace forbidden characters in tag_list tag_dict = {} for tag in tag_list: for c,r in noGood.items(): tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \ if c in tag else tag_dict.get(tag, tag)) # Create output handles handles_dict = {tag:getOutputHandle(db_file, '%s-%s' % (field, label), out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir']) for tag, label in tag_dict.items()} # Create Db writer instances writers_dict = {tag:getDbWriter(handles_dict[tag], db_file) for tag in tag_dict} # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 # Write row to appropriate file tag = row[field] writers_dict[tag].writerow(row) # Sort records into files based on numeric num_split else: num_split = float(num_split) # Create output handles handles_dict = {'under':getOutputHandle(db_file, 'under-%.1f' % num_split, out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir']), 'atleast':getOutputHandle(db_file, 'atleast-%.1f' % num_split, out_type = out_args['out_type'], out_name = out_args['out_name'], out_dir = out_args['out_dir'])} # Create Db writer instances writers_dict = {'under':getDbWriter(handles_dict['under'], db_file), 'atleast':getDbWriter(handles_dict['atleast'], db_file)} # Iterate over IgRecords for row in reader: printProgress(count, rec_count, 0.05, start_time) count += 1 tag = row[field] tag = 'under' if float(tag) < num_split else 'atleast' writers_dict[tag].writerow(row) # Write log printProgress(count, rec_count, 0.05, start_time) log = OrderedDict() for i, k in enumerate(handles_dict): log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name) log['RECORDS'] = rec_count log['PARTS'] = len(handles_dict) log['END'] = 'ParseDb' printLog(log) # Close output file handles for t in handles_dict: handles_dict[t].close() return [handles_dict[t].name for t in handles_dict]
def convertToChangeo(db_file, out_file=None, out_args=default_out_args): """ Converts an AIRR formatted file into an Change-O formatted file Arguments: db_file: the database file name. out_file : output file name. Automatically generated from the input file if None. out_args : common output argument dictionary from parseCommonArgs. Returns: str : output file name. """ log = OrderedDict() log['START'] = 'ConvertDb' log['COMMAND'] = 'changeo' log['FILE'] = os.path.basename(db_file) printLog(log) # Open input db_handle = open(db_file, 'rt') db_iter = AIRRReader(db_handle) # Set output fields replacing length with end fields in_fields = [AIRRSchema.toReceptor(f) for f in db_iter.fields] out_fields = [] for f in in_fields: out_fields.append(f) if f in ReceptorData.end_fields and ReceptorData.end_fields[f][0] in in_fields: out_fields.append(ReceptorData.end_fields[f][1]) out_fields = list(OrderedDict.fromkeys(out_fields)) out_fields = [ChangeoSchema.fromReceptor(f) for f in out_fields] # Open output writer if out_file is not None: pass_handle = open(out_file, 'w') else: pass_handle = getOutputHandle(db_file, out_label='changeo', out_dir=out_args['out_dir'], out_name=out_args['out_name'], out_type=ChangeoSchema.out_type) pass_writer = ChangeoWriter(pass_handle, fields=out_fields) # Count records result_count = countDbFile(db_file) # Iterate over records start_time = time() rec_count = 0 for rec in db_iter: # Print progress for previous iteration printProgress(rec_count, result_count, 0.05, start_time=start_time) rec_count += 1 # Write records pass_writer.writeReceptor(rec) # Print counts printProgress(rec_count, result_count, 0.05, start_time=start_time) log = OrderedDict() log['OUTPUT'] = os.path.basename(pass_handle.name) log['RECORDS'] = rec_count log['END'] = 'ConvertDb' printLog(log) # Close file handles pass_handle.close() db_handle.close() return pass_handle.name
def collectEEQueue(alive, result_queue, collect_queue, seq_file, out_args, set_field): """ Pulls from results queue, assembles results and manages log and file IO Arguments: alive = a multiprocessing.Value boolean controlling whether processing continues; when False function returns result_queue = a multiprocessing.Queue holding worker results collect_queue = a multiprocessing.Queue to store collector return values seq_file = the sample sequence file name out_args = common output argument dictionary from parseCommonArgs set_field = the field defining set membership Returns: None (adds a dictionary of {log: log object, out_files: output file names} to collect_queue) """ try: # Count sets result_count = countSeqSets(seq_file, set_field, out_args['delimiter']) # Define empty DataFrames to store assembled results pos_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float) qual_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float) nuc_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float) set_df = pd.DataFrame(None, columns=['mismatch', 'q_sum', 'total'], dtype=float) # Open log file if out_args['log_file'] is None: log_handle = None else: log_handle = open(out_args['log_file'], 'w') except: alive.value = False raise try: # Iterator over results queue until sentinel object reached start_time = time() set_count = seq_count = pass_count = fail_count = 0 while alive.value: # Get result from queue if result_queue.empty(): continue else: result = result_queue.get() # Exit upon reaching sentinel if result is None: break # Print progress for previous iteration printProgress(set_count, result_count, 0.05, start_time) # Update counts for iteration set_count += 1 seq_count += result.data_count # Sum results if result: pass_count += 1 pos_df = pos_df.add(result.results['pos'], fill_value=0) qual_df = qual_df.add(result.results['qual'], fill_value=0) nuc_df = nuc_df.add(result.results['nuc'], fill_value=0) set_df = set_df.add(result.results['set'], fill_value=0) else: fail_count += 1 # Write log printLog(result.log, handle=log_handle) else: sys.stderr.write('PID %s: Error in sibling process detected. Cleaning up.\n' \ % os.getpid()) return None # Print final progress printProgress(set_count, result_count, 0.05, start_time) # Generate log log = OrderedDict() for i in range(4): log['OUTPUT%i' % (i + 1)] = None log['SETS'] = set_count log['SEQUENCES'] = seq_count log['PASS'] = pass_count log['FAIL'] = fail_count log['POSITION_ERROR'] = None log['NUCLEOTIDE_ERROR'] = None log['QUALITY_ERROR'] = None log['SET_ERROR'] = None # Return if no mismatch data if pass_count == 0: collect_dict = {'log':log, 'out_files': None} collect_queue.put(collect_dict) return None # Calculate error rates pos_df['error'] = pos_df['mismatch'] / pos_df['total'] nuc_df['error'] = nuc_df['mismatch'] / nuc_df['total'] qual_df['error'] = qual_df['mismatch'] / qual_df['total'] set_df['error'] = set_df['mismatch'] / set_df['total'] # Convert error to empirical quality score pos_df['emp_q'] = -10 * np.log10(pos_df['error']) nuc_df['emp_q'] = -10 * np.log10(nuc_df['error']) qual_df['emp_q'] = -10 * np.log10(qual_df['error']) set_df['emp_q'] = -10 * np.log10(set_df['error']) # Calculate reported quality means pos_df['rep_q'] = pos_df['q_sum'] / pos_df['total'] nuc_df['rep_q'] = nuc_df['q_sum'] / nuc_df['total'] qual_df['rep_q'] = qual_df['q_sum'] / qual_df['total'] set_df['rep_q'] = set_df['q_sum'] / set_df['total'] # Calculate overall error rate pos_error = pos_df['mismatch'].sum() / pos_df['total'].sum() qual_error = qual_df['mismatch'].sum() / qual_df['total'].sum() nuc_error = nuc_df['mismatch'].sum() / nuc_df['total'].groupby(level='obs').mean().sum() set_error = set_df['mismatch'].sum() / set_df['total'].sum() # Build results dictionary assembled = {'pos':pos_df, 'qual':qual_df, 'nuc':nuc_df, 'set':set_df} # Write assembled error counts to output files out_files = writeResults(assembled, seq_file, out_args) # Update log for i, f in enumerate(out_files): log['OUTPUT%i' % (i + 1)] = os.path.basename(f) log['POSITION_ERROR'] = pos_error log['NUCLEOTIDE_ERROR'] = nuc_error log['QUALITY_ERROR'] = qual_error log['SET_ERROR'] = set_error # Update collector results collect_dict = {'log':log, 'out_files': out_files} collect_queue.put(collect_dict) except: alive.value = False raise return None