def test_validate(self): # Good data try: result = airr.validate_rearrangement(self.data_good) self.assertTrue(result, 'validate(): good data failed') except: self.assertTrue(False, 'validate(): good data failed') # Bad data try: result = airr.validate_rearrangement(self.data_bad) self.assertFalse(result, 'validate(): bad data failed') except: pass
def test_validate(self): # Good data try: result = airr.validate_rearrangement(self.data_good) self.assertTrue(result, 'validate(): good data failed') except: self.assertTrue(False, 'validate(): good data failed') # Bad data try: result = airr.validate_rearrangement(self.data_bad) self.assertFalse(result, 'validate(): bad data failed') except Exception as inst: print(type(inst)) raise inst
def format_data(args): airr.validate_rearrangement(args.rearrangements, True) reader = airr.read_rearrangement(args.rearrangements) empty_list = [] # keep only the Junction, Vgene, Jgene and Repertoire ID columns keys = ["junction_aa", "v_call", "j_call", "junction", "repertoire_id"] for row in reader: empty_list.append({x: row[x] for x in keys}) df = pd.DataFrame(empty_list) # replace cells without junction with Nan df["junction_aa"].replace("", np.nan, inplace=True) # delete lines with Nan df.dropna(subset=["junction_aa"], inplace=True) # delete lines with an X on the junction_aa df = df[~df.junction_aa.str.contains("X")] # delete lines where junction_aa doesn"t start with C df = df[df.junction_aa.str.startswith("C")] # delete lines where junction_aa doesn"t end with F or W df = df[df.junction_aa.str.endswith(("F", "W"))] # delete lines where the chain in v_call and j_call doesn"t match df = df[(df["v_call"].str[2] == df["j_call"].str[2])] # keep only one first Vgene when there are multiple in the column df["v_call"] = df.v_call.str.split(",", n=1, expand=True)[0] # remove allele information from v_call and keep only the gene information df["v_call"] = df.apply(lambda x: x["v_call"][:-3], axis=1) df["chain"] = df.apply(lambda x: x["v_call"][2], axis=1) return df
if __name__ == '__main__': arguments = docopt(__doc__) #load saved locus information prj_tree = ProjectFolders(os.getcwd()) prj_name = fullpath2last_folder(prj_tree.home) arguments['--rearrangements'] = re.sub("<project>", prj_name, arguments["--rearrangements"]) if not os.path.isfile(arguments['--rearrangements']): sys.exit("Cannot find rearrangments file %s" % arguments['--rearrangements']) elif not airr.validate_rearrangement(arguments['--rearrangements']): sys.exit("File %s is not in valid AIRR format." % arguments['--rearrangements']) if re.match( arguments['--save'], 'all', re.I ): arguments['--save'] = [ "canonical_pair", "heavy_only", "light_only", "possible_inclusion", "multi_light", "multi_heavy", "probable_multiplet" ] elif re.match( arguments['--save'],'good', re.I ): arguments['--save'] = [ "canonical_pair", "heavy_only", "light_only", "possible_inclusion" ] elif re.match( arguments['--save'],'paired', re.I ): arguments['--save'] = [ "canonical_pair" ] else: sys.exit( "%s is not a valid option for --save. Please select from 'all', 'good', or 'paired' only." % arguments['--save'] ) #log command line logCmdLine(sys.argv)
def main(): airrFile = airr.create_rearrangement( "%s/%s_rearrangements.tsv" % (prj_tree.tables, prj_name), fields=[ 'vj_in_frame', 'stop_codon', 'locus', 'c_call', 'junction_length', 'source_file', 'source_id', 'length_raw', 'length_trimmed', 'indels', 'status', 'blast_identity', 'cluster_count', 'v_identity' ]) #try to vacuum up all possible raw sequences and hope it doesn't kill memory raw_seqs = defaultdict(dict) file_list = glob.glob("*.fa") + glob.glob("*.fas") + glob.glob( "*.fst") + glob.glob("*.fasta") + glob.glob("*.fna") + glob.glob( "*.fq") + glob.glob("*.fastq") for myseq, myqual, file_name in generate_read_fasta_folder(file_list): raw_seqs[file_name][myseq.seq_id] = myseq.seq #get trimmed sequences trim_seqs = load_fastas("%s/%s_allJ.fa" % (prj_tree.nt, prj_name)) #get nt junctions junc_seqs = load_fastas("%s/%s_allCDR3.fa" % (prj_tree.nt, prj_name)) #do the conversion with open("%s/%s_all_seq_stats.txt" % (prj_tree.tables, prj_name), "r") as handle: oldFile = csv.reader(handle, delimiter="\t") header = next(oldFile) for row in oldFile: if row[11] == "wrong_length": continue if row[1] not in raw_seqs: sys.stderr.write( "Couldn't find raw sequence file %s, %s will be dropped from converted file.\n" % (row[1], row[0])) continue elif row[2] not in raw_seqs[row[1]]: sys.stderr.write( "Couldn't find raw sequence %s in file %s; %s will be dropped from converted file.\n" % (row[2], row[1], row[0])) continue r = dict() r['sequence'] = raw_seqs[row[1]][row[2]] r['sequence_alignment'] = str( trim_seqs.get(row[0], SeqRecord(seq="")).seq) r['junction'] = str(junc_seqs.get(row[0], SeqRecord(seq="")).seq) r['sequence_id'] = row[0] r['source_file'] = row[1] r['source_id'] = row[2] r['length_raw'] = row[3] if not row[4] == "NA": r['length_trimmed'] = row[4] if not row[5] == "NA": r['v_call'] = row[5] if row[6] not in ["NA", "not_found"]: r['d_call'] = row[6] if not row[7] == "NA": r['j_call'] = row[7] if not row[9] == "NA": r['indels'] = row[9] if not row[10] == "NA": r['stop_codon'] = row[10] r['status'] = row[11] if not row[12] == "NA": r['blast_identity'] = "%.3f" % ( 1 - float(re.sub("%", "", row[12])) / 100) if not row[13] == "NA": r['junction_length'] = int(row[13]) + 6 if not row[15] == "NA": r['junction_aa'] = row[15] if len(row) > 15: if header[16] == "Unique": if row[16] == "T": r['status'] = "unique" r['cluster_count'] = row[17] if len(row) > 17 and not row[18] == "NA": r['v_identity'] = "%.3f" % ( 1 - float(re.sub("%", "", row[18])) / 100) elif header[16] == "V_div" and not row[16] == "NA": r['v_identity'] = "%.3f" % ( 1 - float(re.sub("%", "", row[16])) / 100) #figure out in-frame/productive if row[10] == "good": r['vj_in_frame'] = "T" r['productive'] = "T" elif row[10] == "stop": r['vj_in_frame'] = "T" r['productive'] = "F" elif row[10] == "nonproductive": r['vj_in_frame'] = "F" r['productive'] = "F" elif row[10] == "indel": r['productive'] = "F" #figure out locus if any(x in row[5] for x in ["HV", "VH", "Vh", "vh", "heavy", "Heavy", "HEAVY"]): r['locus'] = "IGH" elif any(x in row[5] for x in ["LV", "VL", "Vl", "vl", "lambda", "Lambda", "LAMBDA"]): r['locus'] = "IGL" elif any(x in row[5] for x in ["KV", "VK", "Vk", "vk", "kappa", "Kappa", "KAPPA"]): r['locus'] = "IGK" airrFile.write(r) airrFile.close() valid = airr.validate_rearrangement("%s/%s_rearrangements.tsv" % (prj_tree.tables, prj_name)) if not valid: sys.exit( "ERROR: something went wrong, %s/%s_rearrangements.tsv failed validation!" % (prj_tree.tables, prj_name))
def main(): airrFile = airr.create_rearrangement( "%s/%s_rearrangements.tsv"%(prj_tree.tables, prj_name), fields=['vj_in_frame','stop_codon','locus','c_call','junction_length','source_file','source_id','length_raw','length_trimmed','indels','status','blast_identity','cluster_count','v_identity']) #try to vacuum up all possible raw sequences and hope it doesn't kill memory raw_seqs = defaultdict( dict ) file_list = glob.glob("*.fa") + glob.glob("*.fas") + glob.glob("*.fst") + glob.glob("*.fasta") + glob.glob("*.fna") + glob.glob("*.fq") + glob.glob("*.fastq") for myseq, myqual, file_name in generate_read_fasta_folder( file_list ): raw_seqs[file_name][myseq.seq_id] = myseq.seq #get trimmed sequences trim_seqs = load_fastas( "%s/%s_allJ.fa"%(prj_tree.nt, prj_name) ) #get nt junctions junc_seqs = load_fastas( "%s/%s_allCDR3.fa"%(prj_tree.nt, prj_name) ) #do the conversion with open( "%s/%s_all_seq_stats.txt"%(prj_tree.tables, prj_name), "r" ) as handle: oldFile = csv.reader( handle, delimiter="\t" ) header = next(oldFile) for row in oldFile: if row[11] == "wrong_length": continue if row[1] not in raw_seqs: sys.stderr.write("Couldn't find raw sequence file %s, %s will be dropped from converted file.\n"%(row[1],row[0])) continue elif row[2] not in raw_seqs[row[1]]: sys.stderr.write("Couldn't find raw sequence %s in file %s; %s will be dropped from converted file.\n"%(row[2],row[1],row[0])) continue r = dict() r['sequence'] = raw_seqs[ row[1] ][ row[2] ] r['sequence_alignment'] = str( trim_seqs.get( row[0], SeqRecord(seq="") ).seq ) r['junction'] = str( junc_seqs.get( row[0], SeqRecord(seq="") ).seq ) r['sequence_id'] = row[0] r['source_file'] = row[1] r['source_id'] = row[2] r['length_raw'] = row[3] if not row[4] == "NA": r['length_trimmed'] = row[4] if not row[5] == "NA": r['v_call'] = row[5] if row[6] not in ["NA", "not_found"]: r['d_call'] = row[6] if not row[7] == "NA": r['j_call'] = row[7] if not row[9] == "NA": r['indels'] = row[9] if not row[10] == "NA": r['stop_codon'] = row[10] r['status'] = row[11] if not row[12] == "NA": r['blast_identity'] = "%.3f" % ( 1 - float(re.sub("%","",row[12]))/100 ) if not row[13] == "NA": r['junction_length'] = int(row[13])+6 if not row[15] == "NA": r['junction_aa'] = row[15] if len(row)>15: if header[16]=="Unique": if row[16] == "T": r['status'] = "unique" r['cluster_count'] = row[17] if len(row)>17 and not row[18]=="NA": r['v_identity'] = "%.3f" % ( 1 - float(re.sub("%","",row[18]))/100 ) elif header[16] == "V_div" and not row[16]=="NA": r['v_identity'] = "%.3f" % ( 1 - float(re.sub("%","",row[16]))/100 ) #figure out in-frame/productive if row[10] == "good": r['vj_in_frame'] = "T" r['productive'] = "T" elif row[10] == "stop": r['vj_in_frame'] = "T" r['productive'] = "F" elif row[10] == "nonproductive": r['vj_in_frame'] = "F" r['productive'] = "F" elif row[10] == "indel": r['productive'] = "F" #figure out locus if any( x in row[5] for x in ["HV", "VH", "Vh", "vh", "heavy", "Heavy", "HEAVY"] ): r['locus'] = "IGH" elif any( x in row[5] for x in ["LV", "VL", "Vl", "vl", "lambda", "Lambda", "LAMBDA"] ): r['locus'] = "IGL" elif any( x in row[5] for x in ["KV", "VK", "Vk", "vk", "kappa", "Kappa", "KAPPA"] ): r['locus'] = "IGK" airrFile.write(r) airrFile.close() valid = airr.validate_rearrangement( "%s/%s_rearrangements.tsv"%(prj_tree.tables, prj_name) ) if not valid: sys.exit( "ERROR: something went wrong, %s/%s_rearrangements.tsv failed validation!"%(prj_tree.tables, prj_name) )
if __name__ == '__main__': arguments = docopt(__doc__) #load saved locus information prj_tree = ProjectFolders(os.getcwd()) prj_name = fullpath2last_folder(prj_tree.home) arguments['--rearrangements'] = re.sub("<project>", prj_name, arguments["--rearrangements"]) if not os.path.isfile(arguments['--rearrangements']): sys.exit("Cannot find rearrangments file %s" % arguments['--rearrangements']) elif not airr.validate_rearrangement(arguments['--rearrangements']): sys.exit("File %s is not in valid AIRR format." % arguments['--rearrangements']) if re.match(arguments['--save'], 'all', re.I): arguments['--save'] = [ "canonical_pair", "heavy_only", "light_only", "possible_inclusion", "multi_light", "multi_heavy", "probable_multiplet" ] elif re.match(arguments['--save'], 'good', re.I): arguments['--save'] = [ "canonical_pair", "heavy_only", "light_only", "possible_inclusion" ] elif re.match(arguments['--save'], 'paired', re.I): arguments['--save'] = ["canonical_pair"] else:
print(mored.external_fields) for r in airr.read_rearrangement('my_data.tsv'): r['new_field'] = 'A' r['more_annotation'] = 'B' print(r) mored.write(r) mored.close() # validate rearrangements file print('*****') print('*****') print('Validate rearrangements file.') print('*****') print('*****') print('Validating more_data.tsv') valid = airr.validate_rearrangement('more_data.tsv') if valid: print('PASS: more_data.tsv passes validation.') else: print('FAIL: more_data.tsv does not pass validation.') # should fail validation due to missing required field print('Validating bad_data.tsv') valid = airr.validate_rearrangement('bad_data.tsv') if not valid: print('PASS: bad_data.tsv fails validation.') else: print('FAIL: bad_data.tsv passed validation.') # merge rearrangements file print('*****')