def __init__(self,sonia_model=None,include_genes=True,processes=None,custom_olga_model=None):
if type(sonia_model)==str or sonia_model is None:
print('ERROR: you need to pass a Sonia object')
return
self.sonia_model=sonia_model
self.include_genes=include_genes
if processes is None: self.processes = mp.cpu_count()
else: self.processes = processes
# define olga model
if custom_olga_model is not None:
self.pgen_model = custom_olga_model
else:
main_folder=os.path.join(os.path.dirname(__file__), 'default_models', self.sonia_model.chain_type)
params_file_name = os.path.join(main_folder,'model_params.txt')
marginals_file_name = os.path.join(main_folder,'model_marginals.txt')
V_anchor_pos_file = os.path.join(main_folder,'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(main_folder,'J_gene_CDR3_anchors.csv')
if self.sonia_model.vj:
genomic_data = olga_load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
self.pgen_model = pgen.GenerationProbabilityVJ(generative_model, genomic_data)
else:
genomic_data = olga_load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
self.pgen_model = pgen.GenerationProbabilityVDJ(generative_model, genomic_data)
def __init__(self,
sonia_model=None,
include_genes=True,
processes=None,
custom_olga_model=None):
if type(sonia_model) == str or sonia_model is None:
print('ERROR: you need to pass a Sonia object')
return
self.sonia_model = sonia_model
self.include_genes = include_genes
# only count usable cpus
# (mp.cpu_count() returns total number of cpus even if not all are available e.g. when running on cluster)
if processes is None: self.processes = len(os.sched_getaffinity(0))
else: self.processes = processes
# define olga model
if custom_olga_model is not None:
self.pgen_model = custom_olga_model
else:
try:
if self.sonia_model.custom_pgen_model is None:
main_folder = os.path.join(os.path.dirname(__file__),
'default_models',
self.sonia_model.chain_type)
else:
main_folder = self.sonia_model.custom_pgen_model
except:
main_folder = os.path.join(os.path.dirname(__file__),
'default_models',
self.sonia_model.chain_type)
params_file_name = os.path.join(main_folder, 'model_params.txt')
marginals_file_name = os.path.join(main_folder,
'model_marginals.txt')
V_anchor_pos_file = os.path.join(main_folder,
'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(main_folder,
'J_gene_CDR3_anchors.csv')
if self.sonia_model.vj:
self.genomic_data = olga_load_model.GenomicDataVJ()
self.genomic_data.load_igor_genomic_data(
params_file_name, V_anchor_pos_file, J_anchor_pos_file)
self.generative_model = olga_load_model.GenerativeModelVJ()
self.generative_model.load_and_process_igor_model(
marginals_file_name)
self.pgen_model = pgen.GenerationProbabilityVJ(
self.generative_model, self.genomic_data)
else:
self.genomic_data = olga_load_model.GenomicDataVDJ()
self.genomic_data.load_igor_genomic_data(
params_file_name, V_anchor_pos_file, J_anchor_pos_file)
self.generative_model = olga_load_model.GenerativeModelVDJ()
self.generative_model.load_and_process_igor_model(
marginals_file_name)
self.pgen_model = pgen.GenerationProbabilityVDJ(
self.generative_model, self.genomic_data)
def compute_pgen(index, seq):
index_ = int(index)
main_folder = os.path.join(local_directory, 'default_models',
options_of[index_])
params_file_name = os.path.join(main_folder, 'model_params.txt')
marginals_file_name = os.path.join(main_folder, 'model_marginals.txt')
V_anchor_pos_file = os.path.join(main_folder, 'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(main_folder, 'J_gene_CDR3_anchors.csv')
if options_of[index] in vj_chains:
genomic_data = olga_load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name,
V_anchor_pos_file,
J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = pgen.GenerationProbabilityVJ(generative_model,
genomic_data)
else:
genomic_data = olga_load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name,
V_anchor_pos_file,
J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = pgen.GenerationProbabilityVDJ(generative_model,
genomic_data)
return pgen_model.compute_aa_CDR3_pgen(seq[0], seq[1],
seq[2]) / norms[index_][0]
def main():
""" Evaluate sequences."""
parser = OptionParser(conflict_handler="resolve")
#specify model
parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
parser.add_option('--humanIGK', '--human_B_kappa', action='store_true', dest='humanIGK', default=False, help='use default human IGK model (B cell light kappa chain)')
parser.add_option('--humanIGL', '--human_B_lambda', action='store_true', dest='humanIGL', default=False, help='use default human IGL model (B cell light lambda chain)')
parser.add_option('--mouseTRA', '--mouse_T_alpha', action='store_true', dest='mouseTRA', default=False, help='use default mouse TRA model (T cell alpha chain)')
parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')
parser.add_option('--sonia_model', type='string', default = 'leftright', dest='model_type' ,help=' specify model type: leftright or lengthpos, default is leftright')
parser.add_option('--ppost', '--Ppost', action='store_true', dest='ppost', default=False, help='compute Ppost, also computes pgen and Q')
parser.add_option('--pgen', '--Pgen', action='store_true', dest='pgen', default=False, help='compute pgen')
parser.add_option('--Q', '--selection_factor', action='store_true', dest='Q', default=False, help='compute Q')
parser.add_option('--recompute_productive_norm', '--compute_norm', action='store_true', dest='recompute_productive_norm', default=False, help='recompute productive normalization')
parser.add_option('--skip_off','--skip_empty_off', action='store_true', dest = 'skip_empty', default=True, help='stop skipping empty or blank sequences/lines (if for example you want to keep line index fidelity between the infile and outfile).')
parser.add_option('-s','--chunk_size', type='int',metavar='N', dest='chunck_size', default = mp.cpu_count()*int(5e2), help='Number of sequences to evaluate at each iteration')
#vj genes
parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', default=None, help='specifies V_masks are found in column INDEX in the input file. Default is None (do not condition on J usage).')
parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', default=None, help='specifies J_masks are found in column INDEX in the input file. Default is None (do not condition on J usage).')
parser.add_option('--v_mask', type='string', dest='V_mask', help='specify V usage to condition as arguments.')
parser.add_option('--j_mask', type='string', dest='J_mask', help='specify J usage to condition as arguments.')
# input output
parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')
parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='evaluate for at most N sequences.')
parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
#delimiters
parser.add_option('-d', '--delimiter', type='choice', dest='delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
parser.add_option('--delimiter_out', type='choice', dest='delimiter_out', choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")
(options, args) = parser.parse_args()
#Check that the model is specified properly
main_folder = os.path.dirname(__file__)
default_models = {}
default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'), 'VJ']
default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']
default_models['humanIGK'] = [os.path.join(main_folder, 'default_models', 'human_B_kappa'), 'VJ']
default_models['humanIGL'] = [os.path.join(main_folder, 'default_models', 'human_B_lambda'), 'VJ']
default_models['mouseTRA'] = [os.path.join(main_folder, 'default_models', 'mouse_T_alpha'), 'VJ']
num_models_specified = sum([1 for x in list(default_models.keys()) + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])
recompute_productive_norm=False
if num_models_specified == 1: #exactly one model specified
try:
d_model = [x for x in default_models.keys() if getattr(options, x)][0]
model_folder = default_models[d_model][0]
recomb_type = default_models[d_model][1]
except IndexError:
if options.vdj_model_folder: #custom VDJ model specified
recompute_productive_norm=True
model_folder = options.vdj_model_folder
recomb_type = 'VDJ'
elif options.vj_model_folder: #custom VJ model specified
recompute_productive_norm=True
model_folder = options.vj_model_folder
recomb_type = 'VJ'
elif num_models_specified == 0:
print('Need to indicate generative model.')
print('Exiting...')
return -1
elif num_models_specified > 1:
print('Only specify one model')
print('Exiting...')
return -1
#Generative model specification -- note we'll probably change this syntax to
#allow for arbitrary model file specification
params_file_name = os.path.join(model_folder,'model_params.txt')
marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')
for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
if not os.path.isfile(x):
print('Cannot find: ' + x)
print('Please check the files (and naming conventions) in the model folder ' + model_folder)
print('Exiting...')
return -1
#Load up model based on recomb_type
#VDJ recomb case --- used for TCRB and IGH
if recomb_type == 'VDJ':
genomic_data = olga_load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data)
#VJ recomb case --- used for TCRA and light chain
elif recomb_type == 'VJ':
genomic_data = olga_load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data)
if options.infile_name is not None:
infile_name = options.infile_name
if not os.path.isfile(infile_name):
print('Cannot find input file: ' + infile_name)
print('Exiting...')
return -1
if options.outfile_name is not None:
outfile_name = options.outfile_name
# if os.path.isfile(outfile_name):
# if not input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
# print('Exiting...')
# return -1
#Parse delimiter
delimiter = options.delimiter
if delimiter is None: #Default case
if options.infile_name is None:
delimiter = '\t'
elif infile_name.endswith('.tsv'): #parse TAB separated value file
delimiter = '\t'
elif infile_name.endswith('.csv'): #parse COMMA separated value file
delimiter = ','
else:
try:
delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#Parse delimiter_out
delimiter_out = options.delimiter_out
if delimiter_out is None: #Default case
if delimiter is None:
delimiter_out = '\t'
else:
delimiter_out = delimiter
if options.outfile_name is None:
pass
elif outfile_name.endswith('.tsv'): #output TAB separated value file
delimiter_out = '\t'
elif outfile_name.endswith('.csv'): #output COMMA separated value file
delimiter_out = ','
else:
try:
delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
except KeyError:
pass #Other string passed as the delimiter.
#Parse gene_delimiter
gene_mask_delimiter = options.gene_mask_delimiter
if gene_mask_delimiter is None: #Default case
gene_mask_delimiter = ','
if delimiter == ',':
gene_mask_delimiter = ';'
else:
try:
gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#More options
seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
lines_to_skip = options.lines_to_skip #one method of skipping header
comment_delimiter = options.comment_delimiter #another method of skipping header
max_number_of_seqs = options.max_number_of_seqs
V_mask_index = options.V_mask_index #Default is not conditioning on V identity
J_mask_index = options.J_mask_index #Default is not conditioning on J identity
skip_empty = options.skip_empty
#print(V_mask_index,J_mask_index,seq_in_index,gene_mask_delimiter,delimiter)
# choose sonia model type
sonia_model=SoniaLeftposRightpos(feature_file=os.path.join(model_folder,'features.tsv'),log_file=os.path.join(model_folder,'log.txt'),vj=recomb_type == 'VJ',custom_pgen_model=model_folder)
if options.recompute_productive_norm:
print('Recompute productive normalization.')
sonia_model.norm_productive=pgen_model.compute_regex_CDR3_template_pgen('CX{0,}')
# load Evaluate model class
ev=EvaluateModel(sonia_model, custom_olga_model=pgen_model,
include_genes=False if ((V_mask_index is None) and (J_mask_index is None)) else True)
if options.infile_name is None: #No infile specified -- args should be the input seq
print_warnings = True
if len(args)>1 :
print('ERROR: can process only one sequence at the time. Submit thourgh file instead.')
return -1
seq=args[0]
#Format V and J masks -- uniform for all argument input sequences
try:
V_mask = options.V_mask.split(',')
unrecognized_v_genes = [v for v in V_mask if gene_to_num_str(v, 'V') not in pgen_model.V_mask_mapping.keys()]
V_mask = [v for v in V_mask if gene_to_num_str(v, 'V') in pgen_model.V_mask_mapping.keys()]
if len(unrecognized_v_genes) > 0:
print('These V genes/alleles are not recognized: ' + ', '.join(unrecognized_v_genes))
if len(V_mask) == 0:
print('No recognized V genes/alleles in the provided V_mask. Continuing without conditioning on V usage.')
V_mask = None
except AttributeError:
V_mask = options.V_mask #Default is None, i.e. not conditioning on V identity
try:
J_mask = options.J_mask.split(',')
unrecognized_j_genes = [j for j in J_mask if gene_to_num_str(j, 'J') not in pgen_model.J_mask_mapping.keys()]
J_mask = [j for j in J_mask if gene_to_num_str(j, 'J') in pgen_model.J_mask_mapping.keys()]
if len(unrecognized_j_genes) > 0:
print('These J genes/alleles are not recognized: ' + ', '.join(unrecognized_j_genes))
if len(J_mask) == 0:
print('No recognized J genes/alleles in the provided J_mask. Continuing without conditioning on J usage.')
J_mask = None
except AttributeError:
J_mask = options.J_mask #Default is None, i.e. not conditioning on J identity
print('')
if options.ppost:
if options.V_mask is None: V_mask=['']
if options.J_mask is None: J_mask=['']
v,j=V_mask[0],J_mask[0]
Q,pgen,ppost=ev.evaluate_seqs([[seq,v,j]])
print('Ppost of ' + seq + ' '+v+ ' '+j+ ': ' + str(ppost[0]))
print('Pgen of ' + seq + ' '+v+ ' '+j+ ': ' + str(pgen[0]))
print('Q of ' + seq + ' '+v+ ' '+j+ ': ' + str(Q[0]))
print('')
elif options.Q:
if options.V_mask is None: V_mask=['']
if options.J_mask is None: J_mask=['']
v,j=V_mask[0],J_mask[0]
Q=ev.evaluate_selection_factors([[seq,v,j]])
print('Q of ' + seq + ' '+v+ ' '+j+ ': ' + str(Q[0]))
elif options.pgen:
pgen=pgen_model.compute_aa_CDR3_pgen(seq,V_mask,J_mask)
if J_mask is None: J_mask= ''
if V_mask is None: V_mask= ''
print('Pgen of ' + seq + ' '+','.join(V_mask)+ ' '+','.join(J_mask)+ ': ' + str(pgen))
else:
print('Specify and option: --ppost, --pgen or --Q')
else:
print('Load file')
seqs = []
V_usage_masks = []
J_usage_masks = []
infile = open(infile_name, 'r')
for i, line in enumerate(infile):
if comment_delimiter is not None: #Default case -- no comments/header delimiter
if line.startswith(comment_delimiter): #allow comments
continue
if i < lines_to_skip:
continue
if delimiter is None: #Default delimiter is any whitespace
split_line = line.split('\n')[0].split()
else:
split_line = line.split('\n')[0].split(delimiter)
#Find the seq
try:
seq = split_line[seq_in_index].strip()
if len(seq.strip()) == 0:
if skip_empty:
continue
else:
seqs.append(seq) #keep the blank seq as a placeholder
#seq_types.append('aaseq')
else:
seqs.append(seq)
#seq_types.append(determine_seq_type(seq, aa_alphabet))
except IndexError: #no index match for seq
if skip_empty and len(line.strip()) == 0:
continue
print('seq_in_index is out of range')
print('Exiting...')
infile.close()
return -1
#Find and format V_usage_mask
if V_mask_index is None:
V_usage_masks.append(['']) #default mask
else:
try:
V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(v, 'V') in pgen_model.V_mask_mapping for v in V_usage_mask]):
V_usage_masks.append(V_usage_mask)
else:
print(str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names")
print('Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if gene_to_num_str(v, 'V') not in pgen_model.V_mask_mapping.keys()]))
print('Exiting...')
infile.close()
return -1
except IndexError: #no index match for V_mask_index
print('V_mask_index is out of range')
print('Exiting...')
infile.close()
return -1
#Find and format J_usage_mask
if J_mask_index is None:
J_usage_masks.append(['']) #default mask
else:
try:
J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(j, 'J') in pgen_model.J_mask_mapping for j in J_usage_mask]):
J_usage_masks.append(J_usage_mask)
else:
print(str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names")
print('Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if gene_to_num_str(j, 'J') not in pgen_model.J_mask_mapping.keys()]))
print('Exiting...')
infile.close()
return -1
except IndexError: #no index match for J_mask_index
print('J_mask_index is out of range')
print('Exiting...')
infile.close()
return -1
if max_number_of_seqs is not None:
if len(seqs) >= max_number_of_seqs:
break
# combine sequences.
zipped=[[seqs[i],V_usage_masks[i][0],J_usage_masks[i][0]] for i in range(len(seqs))]
print('Evaluate')
if options.outfile_name is not None: #OUTFILE SPECIFIED
with open(options.outfile_name,'w') as file:
if options.ppost:file.write('Q'+delimiter_out+'Pgen'+delimiter_out+'Ppost\n')
elif options.Q:file.write('Q\n')
elif options.pgen:file.write('Pgen\n')
else:
print('Specify one option: --ppost, --pgen or --Q')
return -1
for t in tqdm(chunks(zipped,options.chunck_size)):
if options.ppost:
Q,pgen,ppost=ev.evaluate_seqs(t)
for i in range(len(Q)):file.write(str(Q[i])+delimiter_out+str(pgen[i])+delimiter_out+str(ppost[i])+'\n')
elif options.Q:
Q=ev.evaluate_selection_factors(t)
for i in range(len(Q)):file.write(str(Q[i])+'\n')
elif options.pgen:
pgens=ev.compute_all_pgens(t)
for i in range(len(pgens)):file.write(str(pgens[i])+'\n')
else: #print to stdout
for t in chunks(zipped,options.chunck_size):
if options.ppost:
Q,pgen,ppost=ev.evaluate_seqs(t)
print ('Q, Pgen, Ppost')
for i in range(len(Q)):print(Q[i],pgen[i],ppost[i])
elif options.Q:
Q=ev.evaluate_selection_factors(t)
print ('Q')
print(Q)
elif options.pgen:
pgens=ev.compute_all_pgens(t)
print ('Pgen')
print(pgens)
else:
print('Specify one option: --ppost, --pgen or --Q')
def __init__(self, chain_folder, recomb_type):
"""
Sets up an OlgaModel that can be used multiple times. For instance to
generate generation probabilities for 10K sequences.
chain_folder : string
'human_T_beta', 'human_T_alpha'
recomb_type : string
'VDJ' or "VJ"
"""
self.chain_folder = chain_folder
self.recomb_type = recomb_type
self.generative_model = None
self.genomic_data = None
self.pgen_model = None
self.seq_gen_model = None
self._validate_chain_folder_arg()
self._validate_recomb_type_arg()
self._validate_chain_folder_with_recomb_type()
params_file_name = op.join(path_to_olga_default_models, chain_folder,
'model_params.txt')
marginals_file_name = op.join(path_to_olga_default_models,
chain_folder, 'model_marginals.txt')
V_anchor_pos_file = op.join(path_to_olga_default_models, chain_folder,
'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = op.join(path_to_olga_default_models, chain_folder,
'J_gene_CDR3_anchors.csv')
#Load up model based on recomb_type
#VDJ recomb case --- used for TCRB and IGH
if recomb_type == 'VDJ':
genomic_data = load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name,
V_anchor_pos_file,
J_anchor_pos_file)
generative_model = load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVDJ(
generative_model, genomic_data)
self.genomic_data = genomic_data
self.generative_model = generative_model
self.pgen_model = pgen_model
self.seq_gen_model = seq_gen.SequenceGenerationVDJ(
self.generative_model, self.genomic_data)
#VJ recomb case --- used for TCRA and light chain
elif recomb_type == 'VJ':
genomic_data = load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name,
V_anchor_pos_file,
J_anchor_pos_file)
generative_model = load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVJ(
generative_model, genomic_data)
self.genomic_data = genomic_data
self.generative_model = generative_model
self.pgen_model = pgen_model
self.seq_gen_model = seq_gen.SequenceGenerationVJ(
self.generative_model, self.genomic_data)
def main():
""" Evaluate sequences."""
parser = OptionParser(conflict_handler="resolve")
#specify model
parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
parser.add_option('--humanIGK', '--human_B_kappa', action='store_true', dest='humanIGK', default=False, help='use default human IGK model (B cell light kappa chain)')
parser.add_option('--humanIGL', '--human_B_lambda', action='store_true', dest='humanIGL', default=False, help='use default human IGL model (B cell light lambda chain)')
parser.add_option('--mouseTRA', '--mouse_T_alpha', action='store_true', dest='mouseTRA', default=False, help='use default mouse TRA model (T cell alpha chain)')
parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')
parser.add_option('--sonia_model', type='string', default = 'leftright', dest='model_type' ,help='specify model type: leftright or lengthpos, default is leftright')
parser.add_option('--epochs', type='int', default = 30, dest='epochs' ,help='number of epochs for inference, default is 30')
parser.add_option('--batch_size', type='int', default = 5000, dest='batch_size' ,help='size of batch for the stochastic gradient descent')
parser.add_option('--validation_split', type='float', default = 0.2, dest='validation_split' ,help='fraction of sequences used for validation.')
parser.add_option('--independent_genes', '--include_indep_genes', action='store_true', dest='independent_genes', default=False, help='Independent gene selection factors q_v*q_j. Deafult is joint q_vj')
parser.add_option('--min_energy_clip', type='float', default=-5, dest='min_energy_clip', help='Set numerical lower bound to the values of -logQ, default is -5.')
parser.add_option('--max_energy_clip', type='float', default=10, dest='max_energy_clip', help='Set numerical upper bound to the values of -logQ, default is 10.')
#location of seqs
parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')
parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', default=1, help='specifies V_masks are found in column INDEX in the input file. Default is 1.')
parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', default=2, help='specifies J_masks are found in column INDEX in the input file. Default is 2.')
# input output
parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='evaluate for at most N sequences.')
parser.add_option('-n', '--n_gen_seqs', type='int',metavar='N', dest='n_gen_seqs',default=0, help='sample n sequences from gen distribution.')
parser.add_option('-g', '--infile_gen', dest = 'infile_gen',metavar='PATH/TO/FILE', help='read generated CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
parser.add_option('--no_report', '--no_plot_report', action='store_false', dest='plot_report', default=True, help='Do not produce report plots of the inferred model.')
#delimeters
parser.add_option('-d', '--delimiter', type='choice', dest='delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
parser.add_option('--delimiter_out', type='choice', dest='delimiter_out', choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")
parser.add_option('--seed', type='int',metavar='N', dest='seed', default = None, help='set seed for inference')
(options, args) = parser.parse_args()
#set seed
if options.seed is not None:
import tensorflow as tf
np.random.seed(options.seed)
tf.random.set_seed(options.seed)
#Check that the model is specified properly
main_folder = os.path.dirname(__file__)
default_models = {}
default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'), 'VJ']
default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']
default_models['humanIGK'] = [os.path.join(main_folder, 'default_models', 'human_B_kappa'), 'VJ']
default_models['humanIGL'] = [os.path.join(main_folder, 'default_models', 'human_B_lambda'), 'VJ']
default_models['mouseTRA'] = [os.path.join(main_folder, 'default_models', 'mouse_T_alpha'), 'VJ']
if options.independent_genes:
independent_genes=True
joint_genes=False
else:
independent_genes=False
joint_genes=True
num_models_specified = sum([1 for x in list(default_models.keys()) + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])
recompute_productive_norm=False
if num_models_specified == 1: #exactly one model specified
try:
d_model = [x for x in default_models.keys() if getattr(options, x)][0]
model_folder = default_models[d_model][0]
recomb_type = default_models[d_model][1]
except IndexError:
if options.vdj_model_folder: #custom VDJ model specified
recompute_productive_norm=True
model_folder = options.vdj_model_folder
recomb_type = 'VDJ'
elif options.vj_model_folder: #custom VJ model specified
recompute_productive_norm=True
model_folder = options.vj_model_folder
recomb_type = 'VJ'
elif num_models_specified == 0:
print('Need to indicate generative model.')
print('Exiting...')
return -1
elif num_models_specified > 1:
print('Only specify one model')
print('Exiting...')
return -1
if options.max_energy_clip <= options.min_energy_clip :
print('The clip for the higher energy must be strictly greater than the clip for the lower energy. ')
print('Exiting...')
return -1
else :
max_energy_clip = options.max_energy_clip
min_energy_clip = options.min_energy_clip
#Generative model specification -- note we'll probably change this syntax to
#allow for arbitrary model file specification
params_file_name = os.path.join(model_folder,'model_params.txt')
marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')
for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
if not os.path.isfile(x):
print('Cannot find: ' + x)
print('Please check the files (and naming conventions) in the model folder ' + model_folder)
print('Exiting...')
return -1
#Load up model based on recomb_type
#VDJ recomb case --- used for TCRB and IGH
if recomb_type == 'VDJ':
genomic_data = olga_load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data)
#VJ recomb case --- used for TCRA and light chain
elif recomb_type == 'VJ':
genomic_data = olga_load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = olga_load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data)
if options.infile_name is not None:
infile_name = options.infile_name
if not os.path.isfile(infile_name):
print('Cannot find input file: ' + infile_name)
print('Exiting...')
return -1
if options.outfile_name is not None:
outfile_name = options.outfile_name
if os.path.isfile(outfile_name):
if not input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
print('Exiting...')
return -1
#Parse delimiter
delimiter = options.delimiter
if delimiter is None: #Default case
if options.infile_name is None:
delimiter = '\t'
elif infile_name.endswith('.tsv'): #parse TAB separated value file
delimiter = '\t'
elif infile_name.endswith('.csv'): #parse COMMA separated value file
delimiter = ','
else:
try:
delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#Parse delimiter_out
delimiter_out = options.delimiter_out
if delimiter_out is None: #Default case
if delimiter is None:
delimiter_out = '\t'
else:
delimiter_out = delimiter
if options.outfile_name is None:
pass
elif outfile_name.endswith('.tsv'): #output TAB separated value file
delimiter_out = '\t'
elif outfile_name.endswith('.csv'): #output COMMA separated value file
delimiter_out = ','
else:
try:
delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
except KeyError:
pass #Other string passed as the delimiter.
#Parse gene_delimiter
gene_mask_delimiter = options.gene_mask_delimiter
if gene_mask_delimiter is None: #Default case
gene_mask_delimiter = ','
if delimiter == ',':
gene_mask_delimiter = ';'
else:
try:
gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#More options
seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
lines_to_skip = options.lines_to_skip #one method of skipping header
comment_delimiter = options.comment_delimiter #another method of skipping header
max_number_of_seqs = options.max_number_of_seqs
V_mask_index = options.V_mask_index #Default is not conditioning on V identity
J_mask_index = options.J_mask_index #Default is not conditioning on J identity
skip_empty=True # skip empty lines
if options.infile_name is None: #No infile specified -- args should be the input seqs
print('ERROR: specify input file.')
return -1
else:
seqs = []
V_usage_masks = []
J_usage_masks = []
print('Read input file.')
infile = open(infile_name, 'r')
for i, line in enumerate(tqdm(infile)):
if comment_delimiter is not None: #Default case -- no comments/header delimiter
if line.startswith(comment_delimiter): #allow comments
continue
if i < lines_to_skip:
continue
if delimiter is None: #Default delimiter is any whitespace
split_line = line.split('\n')[0].split()
else:
split_line = line.split('\n')[0].split(delimiter)
#Find the seq
try:
seq = split_line[seq_in_index].strip()
if len(seq.strip()) == 0:
if skip_empty:
continue
else:
seqs.append(seq) #keep the blank seq as a placeholder
#seq_types.append('aaseq')
else:
seqs.append(seq)
#seq_types.append(determine_seq_type(seq, aa_alphabet))
except IndexError: #no index match for seq
if skip_empty and len(line.strip()) == 0:
continue
print('seq_in_index is out of range')
print('Exiting...')
infile.close()
return -1
#Find and format V_usage_mask
if V_mask_index is None:
V_usage_masks.append(None) #default mask
else:
try:
V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(v, 'V') in pgen_model.V_mask_mapping for v in V_usage_mask]):
V_usage_masks.append(V_usage_mask)
else:
print(str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names")
print('Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if gene_to_num_str(v, 'V') not in pgen_model.V_mask_mapping.keys()]))
print('Continuing but inference might be biased...')
V_usage_masks.append(V_usage_mask)
#infile.close()
#return -1
except IndexError: #no index match for V_mask_index
print('V_mask_index is out of range, check the delimeter.')
print('Exiting...')
infile.close()
return -1
#Find and format J_usage_mask
if J_mask_index is None:
J_usage_masks.append(None) #default mask
else:
try:
J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(j, 'J') in pgen_model.J_mask_mapping for j in J_usage_mask]):
J_usage_masks.append(J_usage_mask)
else:
print(str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names")
print('Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if gene_to_num_str(j, 'J') not in pgen_model.J_mask_mapping.keys()]))
print('Continuing but inference might be biased...')
J_usage_masks.append(J_usage_mask)
#infile.close()
#return -1
except IndexError: #no index match for J_mask_index
print('J_mask_index is out of range, check the delimeter.')
print('Exiting...')
infile.close()
return -1
if max_number_of_seqs is not None:
if len(seqs) >= max_number_of_seqs:
break
data_seqs=[[seqs[i],V_usage_masks[i][0],J_usage_masks[i][0]] for i in range(len(seqs))]
#define number of gen_seqs:
gen_seqs=[]
n_gen_seqs=options.n_gen_seqs
generate_sequences=False
if options.infile_gen is None:
generate_sequences=True
if n_gen_seqs is 0: n_gen_seqs=np.max([int(3e5),3*len(data_seqs)])
else:
seqs = []
V_usage_masks = []
J_usage_masks = []
print('Read file of generated seqs.')
infile = open(options.infile_gen, 'r')
for i, line in enumerate(tqdm(infile)):
if comment_delimiter is not None: #Default case -- no comments/header delimiter
if line.startswith(comment_delimiter): #allow comments
continue
if i < lines_to_skip:
continue
if delimiter is None: #Default delimiter is any whitespace
split_line = line.split('\n')[0].split()
else:
split_line = line.split('\n')[0].split(delimiter)
#Find the seq
try:
seq = split_line[seq_in_index].strip()
if len(seq.strip()) == 0:
if skip_empty:
continue
else:
seqs.append(seq) #keep the blank seq as a placeholder
#seq_types.append('aaseq')
else:
seqs.append(seq)
#seq_types.append(determine_seq_type(seq, aa_alphabet))
except IndexError: #no index match for seq
if skip_empty and len(line.strip()) == 0:
continue
print('seq_in_index is out of range')
print('Exiting...')
infile.close()
return -1
#Find and format V_usage_mask
if V_mask_index is None:
V_usage_masks.append(None) #default mask
else:
try:
V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(v, 'V') in pgen_model.V_mask_mapping for v in V_usage_mask]):
V_usage_masks.append(V_usage_mask)
else:
print(str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names")
print('Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if gene_to_num_str(v, 'V') not in pgen_model.V_mask_mapping.keys()]))
print('Continuing but inference might be biased...')
V_usage_masks.append(V_usage_mask)
#infile.close()
#return -1
except IndexError: #no index match for V_mask_index
print('V_mask_index is out of range, check the delimeter.')
print('Exiting...')
infile.close()
return -1
#Find and format J_usage_mask
if J_mask_index is None:
J_usage_masks.append(None) #default mask
else:
try:
J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([gene_to_num_str(j, 'J') in pgen_model.J_mask_mapping for j in J_usage_mask]):
J_usage_masks.append(J_usage_mask)
else:
print(str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names")
print('Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if gene_to_num_str(j, 'J') not in pgen_model.J_mask_mapping.keys()]))
print('Continuing but inference might be biased...')
J_usage_masks.append(J_usage_mask)
#infile.close()
#return -1
except IndexError: #no index match for J_mask_index
print('J_mask_index is out of range, check the delimeter.')
print('Exiting...')
infile.close()
return -1
gen_seqs=[[seqs[i],V_usage_masks[i][0],J_usage_masks[i][0]] for i in range(len(seqs))]
# combine sequences.
print('Initialise Model.')
# choose sonia model type
if options.model_type=='leftright':
sonia_model=SoniaLeftposRightpos(data_seqs=data_seqs,
gen_seqs=gen_seqs,
custom_pgen_model=model_folder,
vj=recomb_type == 'VJ',
include_joint_genes=joint_genes,
include_indep_genes=independent_genes,
min_energy_clip=min_energy_clip,
max_energy_clip=max_energy_clip
)
elif options.model_type=='lengthpos':
sonia_model=SoniaLengthPos(data_seqs=data_seqs,
gen_seqs=gen_seqs,
custom_pgen_model=model_folder,
vj=recomb_type == 'VJ',
include_joint_genes=joint_genes,
include_indep_genes=independent_genes,
min_energy_clip=min_energy_clip,
max_energy_clip=max_energy_clip
)
else:
print('ERROR: choose a model between leftright or lengthpos')
if generate_sequences: sonia_model.add_generated_seqs(n_gen_seqs,custom_model_folder=model_folder)
if recompute_productive_norm: sonia_model.norm_productive=pgen_model.compute_regex_CDR3_template_pgen('CX{0,}')
print('Model initialised. Start inference')
sonia_model.infer_selection(epochs=options.epochs,verbose=1,batch_size=options.batch_size,validation_split=options.validation_split)
print('Save Model')
if options.outfile_name is not None: #OUTFILE SPECIFIED
sonia_model.save_model(options.outfile_name)
if options.plot_report:
from sonia.plotting import Plotter
pl=Plotter(sonia_model)
pl.plot_model_learning(os.path.join(options.outfile_name, 'model_learning.png'))
pl.plot_vjl(os.path.join(options.outfile_name, 'marginals.png'))
pl.plot_logQ(os.path.join(options.outfile_name, 'log_Q.png'))
pl.plot_ratioQ(os.path.join(options.outfile_name, 'Q_ratio.png'))
else: #print to stdout
sonia_model.save_model('sonia_model')
if options.plot_report:
from sonia.plotting import Plotter
pl=Plotter(sonia_model)
pl.plot_model_learning(os.path.join('sonia_model', 'model_learning.png'))
pl.plot_vjl(os.path.join('sonia_model', 'marginals.png'))
pl.plot_logQ(os.path.join('sonia_model', 'log_Q.png'))
pl.plot_ratioQ(os.path.join('sonia_model', 'Q_ratio.png'))
def main():
"""Compute Pgens from a file and output to another file."""
parser = OptionParser(conflict_handler="resolve")
parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')
parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')
parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', help='specifies V_masks are found in column INDEX in the input file. Default is no V mask.')
parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', help='specifies J_masks are found in column INDEX in the input file. Default is no J mask.')
parser.add_option('--v_mask', type='string', dest='V_mask', help='specify V usage to condition Pgen on for seqs read in as arguments.')
parser.add_option('--j_mask', type='string', dest='J_mask', help='specify J usage to condition Pgen on for seqs read in as arguments.')
parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='compute Pgens for at most N sequences.')
parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
parser.add_option('-a', '--alphabet_filename', dest='alphabet_filename', metavar='PATH/TO/FILE', help="specify PATH/TO/FILE defining a custom 'amino acid' alphabet. Default is no custom alphabet.")
parser.add_option('--seq_type_out', type='choice',metavar='SEQ_TYPE', dest='seq_type_out', choices=['all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'], help="if read in sequences are ntseqs, declare what type of sequence to compute pgen for. Default is all. Choices: 'all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'")
parser.add_option('--skip_off','--skip_empty_off', action='store_true', dest = 'skip_empty', default=True, help='stop skipping empty or blank sequences/lines (if for example you want to keep line index fidelity between the infile and outfile).')
parser.add_option('--display_off', action='store_false', dest='display_seqs', default=True, help='turn the sequence display off (only applies in write-to-file mode). Default is on.')
parser.add_option('--num_lines_for_display', type='int', metavar='N', default = 50, dest='num_lines_for_display', help='N lines of the output file are displayed when sequence display is on. Also used to determine the number of sequences to average over for speed and time estimates.')
parser.add_option('--time_updates_off', action='store_false', dest='time_updates', default=True, help='turn time updates off (only applies when sequence display is disabled).')
parser.add_option('--seqs_per_time_update', type='float', metavar='N', default = 100, dest='seqs_per_time_update', help='specify the number of sequences between time updates. Default is 1e5.')
parser.add_option('-d', '--delimiter', type='choice', dest='delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
parser.add_option('--delimiter_out', type='choice', dest='delimiter_out', choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter', choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")
(options, args) = parser.parse_args()
#Check that the model is specified properly
main_folder = os.path.dirname(__file__)
default_models = {}
default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'), 'VJ']
default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']
num_models_specified = sum([1 for x in default_models.keys() + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])
if num_models_specified == 1: #exactly one model specified
try:
d_model = [x for x in default_models.keys() if getattr(options, x)][0]
model_folder = default_models[d_model][0]
recomb_type = default_models[d_model][1]
except IndexError:
if options.vdj_model_folder: #custom VDJ model specified
model_folder = options.vdj_model_folder
recomb_type = 'VDJ'
elif options.vj_model_folder: #custom VJ model specified
model_folder = options.vj_model_folder
recomb_type = 'VJ'
elif num_models_specified == 0:
print 'Need to indicate generative model.'
print 'Exiting...'
return -1
elif num_models_specified > 1:
print 'Only specify one model'
print 'Exiting...'
return -1
#Check that all model and genomic files exist in the indicated model folder
if not os.path.isdir(model_folder):
print 'Check pathing... cannot find the model folder: ' + model_folder
print 'Exiting...'
return -1
params_file_name = os.path.join(model_folder,'model_params.txt')
marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')
for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
if not os.path.isfile(x):
print 'Cannot find: ' + x
print 'Please check the files (and naming conventions) in the model folder ' + model_folder
print 'Exiting...'
return -1
alphabet_filename = options.alphabet_filename #used if a custom alphabet is to be specified
if alphabet_filename is not None:
if not os.path.isfile(alphabet_filename):
print 'Cannot find custom alphabet file: ' + infile_name
print 'Exiting...'
return -1
#Load up model based on recomb_type
#VDJ recomb case --- used for TCRB and IGH
if recomb_type == 'VDJ':
genomic_data = load_model.GenomicDataVDJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = load_model.GenerativeModelVDJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data, alphabet_filename)
#VJ recomb case --- used for TCRA and light chain
elif recomb_type == 'VJ':
genomic_data = load_model.GenomicDataVJ()
genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
generative_model = load_model.GenerativeModelVJ()
generative_model.load_and_process_igor_model(marginals_file_name)
pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data, alphabet_filename)
aa_alphabet = ''.join(pgen_model.codons_dict.keys())
if options.infile_name is not None:
infile_name = options.infile_name
if not os.path.isfile(infile_name):
print 'Cannot find input file: ' + infile_name
print 'Exiting...'
return -1
if options.outfile_name is not None:
outfile_name = options.outfile_name
if os.path.isfile(outfile_name):
if not raw_input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
print 'Exiting...'
return -1
#Parse delimiter
delimiter = options.delimiter
if delimiter is None: #Default case
if options.infile_name is None:
delimiter = '\t'
elif infile_name.endswith('.tsv'): #parse TAB separated value file
delimiter = '\t'
elif infile_name.endswith('.csv'): #parse COMMA separated value file
delimiter = ','
else:
try:
delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#Parse delimiter_out
delimiter_out = options.delimiter_out
if delimiter_out is None: #Default case
if delimiter is None:
delimiter_out = '\t'
else:
delimiter_out = delimiter
if options.outfile_name is None:
pass
elif outfile_name.endswith('.tsv'): #output TAB separated value file
delimiter_out = '\t'
elif outfile_name.endswith('.csv'): #output COMMA separated value file
delimiter_out = ','
else:
try:
delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
except KeyError:
pass #Other string passed as the delimiter.
#Parse gene_delimiter
gene_mask_delimiter = options.gene_mask_delimiter
if gene_mask_delimiter is None: #Default case
gene_mask_delimiter = ','
if delimiter == ',':
gene_mask_delimiter = ';'
else:
try:
gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
except KeyError:
pass #Other string passed as the delimiter.
#More options
time_updates = options.time_updates
display_seqs = options.display_seqs
num_lines_for_display = options.num_lines_for_display
seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
lines_to_skip = options.lines_to_skip #one method of skipping header
comment_delimiter = options.comment_delimiter #another method of skipping header
seqs_per_time_update = options.seqs_per_time_update
max_number_of_seqs = options.max_number_of_seqs
V_mask_index = options.V_mask_index #Default is not conditioning on V identity
J_mask_index = options.J_mask_index #Default is not conditioning on J identity
skip_empty = options.skip_empty
seq_type_out = options.seq_type_out #type of pgens to be computed. Can be ntseq, aaseq, or both
if seq_type_out is not None:
seq_type_out = {'all': None, 'ntseq': 'ntseq', 'nucleotide': 'ntseq', 'aaseq': 'aaseq', 'amino_acid': 'aaseq'}[seq_type_out]
if options.infile_name is None: #No infile specified -- args should be the input seqs
print_warnings = True
seqs = args
seq_types = [determine_seq_type(seq, aa_alphabet) for seq in seqs]
unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
if len(unrecognized_seqs) > 0 and print_warnings:
print 'The following sequences/arguments were not recognized: ' + ', '.join(unrecognized_seqs)
seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
seq_types = [seq_type for seq_type in seq_types if seq_type is not None]
#Format V and J masks -- uniform for all argument input sequences
try:
V_mask = options.V_mask.split(',')
unrecognized_v_genes = [v for v in V_mask if v not in pgen_model.V_mask_mapping.keys()]
V_mask = [v for v in V_mask if v in pgen_model.V_mask_mapping.keys()]
if len(unrecognized_v_genes) > 0:
print 'These V genes/alleles are not recognized: ' + ', '.join(unrecognized_v_genes)
if len(V_mask) == 0:
print 'No recognized V genes/alleles in the provided V_mask. Continuing without conditioning on V usage.'
V_mask = None
except AttributeError:
V_mask = options.V_mask #Default is None, i.e. not conditioning on V identity
try:
J_mask = options.J_mask.split(',')
unrecognized_j_genes = [j for j in J_mask if j not in pgen_model.J_mask_mapping.keys()]
J_mask = [j for j in J_mask if j in pgen_model.J_mask_mapping.keys()]
if len(unrecognized_j_genes) > 0:
print 'These J genes/alleles are not recognized: ' + ', '.join(unrecognized_j_genes)
if len(J_mask) == 0:
print 'No recognized J genes/alleles in the provided J_mask. Continuing without conditioning on J usage.'
J_mask = None
except AttributeError:
J_mask = options.J_mask #Default is None, i.e. not conditioning on J identity
print ''
start_time = time.time()
for seq, seq_type in zip(seqs, seq_types):
if seq_type == 'aaseq':
c_pgen = pgen_model.compute_aa_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
print 'Pgen of the amino acid sequence ' + seq + ': ' + str(c_pgen)
print ''
elif seq_type == 'regex':
c_pgen = pgen_model.compute_regex_CDR3_template_pgen(seq, V_mask, J_mask, print_warnings)
print 'Pgen of the regular expression sequence ' + seq + ': ' + str(c_pgen)
print ''
elif seq_type == 'ntseq':
if seq_type_out is None or seq_type_out == 'ntseq':
c_pgen_nt = pgen_model.compute_nt_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
print 'Pgen of the nucleotide sequence ' + seq + ': ' + str(c_pgen_nt)
if seq_type_out is None or seq_type_out == 'aaseq':
c_pgen_aa = pgen_model.compute_aa_CDR3_pgen(nt2aa(seq), V_mask, J_mask, print_warnings)
print 'Pgen of the amino acid sequence nt2aa(' + seq + ') = ' + nt2aa(seq) + ': ' + str(c_pgen_aa)
print ''
c_time = time.time() - start_time
if c_time > 86400: #more than a day
c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 3600: #more than an hr
c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 60: #more than a min
c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
else:
c_time_str = '%.2f seconds.'%(c_time)
print 'Completed pgen computation in: ' + c_time_str
else: #Read sequences in from file
print_warnings = False #Most cases of reading in from file should have warnings disabled
seqs = []
seq_types = []
V_usage_masks = []
J_usage_masks = []
infile = open(infile_name, 'r')
for i, line in enumerate(infile):
if comment_delimiter is not None: #Default case -- no comments/header delimiter
if line.startswith(comment_delimiter): #allow comments
continue
if i < lines_to_skip:
continue
if delimiter is None: #Default delimiter is any whitespace
split_line = line.split()
else:
split_line = line.split(delimiter)
#Find the seq
try:
seq = split_line[seq_in_index].strip()
if len(seq.strip()) == 0:
if skip_empty:
continue
else:
seqs.append(seq) #keep the blank seq as a placeholder
seq_types.append('aaseq')
else:
seqs.append(seq)
seq_types.append(determine_seq_type(seq, aa_alphabet))
except IndexError: #no index match for seq
if skip_empty and len(line.strip()) == 0:
continue
print 'seq_in_index is out of range'
print 'Exiting...'
infile.close()
return -1
#Find and format V_usage_mask
if V_mask_index is None:
V_usage_masks.append(None) #default mask
else:
try:
V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([v in pgen_model.V_mask_mapping for v in V_usage_mask]):
V_usage_masks.append(V_usage_mask)
else:
print str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names"
print 'Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if not v in pgen_model.V_mask_mapping.keys()])
print 'Exiting...'
infile.close()
return -1
except IndexError: #no index match for V_mask_index
print 'V_mask_index is out of range'
print 'Exiting...'
infile.close()
return -1
#Find and format J_usage_mask
if J_mask_index is None:
J_usage_masks.append(None) #default mask
else:
try:
J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
#check that all V gene/allele names are recognized
if all([j in pgen_model.J_mask_mapping for j in J_usage_mask]):
J_usage_masks.append(J_usage_mask)
else:
print str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names"
print 'Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if not j in pgen_model.J_mask_mapping.keys()])
print 'Exiting...'
infile.close()
return -1
except IndexError: #no index match for J_mask_index
print 'J_mask_index is out of range'
print 'Exiting...'
infile.close()
return -1
if max_number_of_seqs is not None:
if len(seqs) >= max_number_of_seqs:
break
unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
if len(unrecognized_seqs) > 0 and len(unrecognized_seqs) < len(seqs):
if print_warnings or options.outfile_name is not None:
print 'Some strings read in were not parsed as sequences -- they will be omitted.'
print 'Examples of improperly read strings: '
for unrecognized_seq in unrecognized_seqs[:10]:
print unrecognized_seq
seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
V_usage_masks = [V_usage_mask for i, V_usage_mask in enumerate(V_usage_masks) if seq_types[i] is not None]
seq_types = [seq_type for seq_type in seq_types if seq_type is not None]
elif len(unrecognized_seqs) > 0 and len(unrecognized_seqs) == len(seqs):
print 'None of the read in strings were parsed as sequences. Check input file.'
print 'Examples of improperly read strings:'
for unrecognized_seq in unrecognized_seqs[:10]:
print unrecognized_seq
print 'Exiting...'
return -1
infile.close()
if options.outfile_name is not None: #OUTFILE SPECIFIED, allow printed info/display
print 'Successfully read in and formatted ' + str(len(seqs)) + ' sequences and any V or J usages.'
if display_seqs:
sys.stdout.write('\r'+'Continuing to Pgen computation in 3... ')
sys.stdout.flush()
time.sleep(0.4)
sys.stdout.write('\r'+'Continuing to Pgen computation in 2... ')
sys.stdout.flush()
time.sleep(0.4)
sys.stdout.write('\r'+'Continuing to Pgen computation in 1... ')
sys.stdout.flush()
time.sleep(0.4)
else:
print 'Continuing to Pgen computation.'
print_warnings = True #Display is off, can print warnings
if display_seqs:
lines_for_display = []
times_for_speed_calc = [time.time()]
outfile = open(outfile_name, 'w')
start_time = time.time()
for i, seq in enumerate(seqs):
if seq_types[i] == 'aaseq':
#Compute Pgen and print out
c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
if seq_types[i] == 'regex':
#Compute Pgen and print out
c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_types[i] == 'ntseq':
ntseq = seq
if len(ntseq) % 3 == 0: #inframe sequence
aaseq = nt2aa(ntseq)
#Compute Pgen and print out based on recomb_type and seq_type_out
if seq_type_out is None:
c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_type_out == 'ntseq':
c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_type_out == 'aaseq':
c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
if seq_type_out is None:
c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
elif seq_type_out == 'ntseq':
c_pgen_line = ntseq + delimiter_out + '0'
elif seq_type_out == 'aaseq':
c_pgen_line = 'out_of_frame' + delimiter_out + '0'
outfile.write(c_pgen_line + '\n')
#Print time update
if display_seqs:
cc_time = time.time()
c_time = cc_time - start_time
times_for_speed_calc = [cc_time] + times_for_speed_calc[:num_lines_for_display]
c_avg_speed = (len(times_for_speed_calc)-1)/float(times_for_speed_calc[0] - times_for_speed_calc[-1])
#eta = ((len(seqs) - (i+1))/float(i+1))*c_time
eta = (len(seqs) - (i+1))/c_avg_speed
lines_for_display = [c_pgen_line] + lines_for_display[:num_lines_for_display]
c_time_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(c_time)/3600).rjust(3), repr((int(c_time)/60)%60).rjust(2), repr(int(c_time)%60).rjust(2))
eta_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(eta)/3600).rjust(3), repr((int(eta)/60)%60).rjust(2), repr(int(eta)%60).rjust(2))
time_str = 'Time to compute Pgen on %s seqs: %s \nEst. time for remaining %s seqs: %s'%(repr(i+1).rjust(9), c_time_str, repr(len(seqs) - (i + 1)).rjust(9), eta_str)
speed_str = 'Current Pgen computation speed: %s seqs/min'%(repr(round((len(times_for_speed_calc)-1)*60/float(times_for_speed_calc[0] - times_for_speed_calc[-1]), 2)).rjust(8))
display_str = '\n'.join(lines_for_display[::-1]) + '\n' + '-'*80 + '\n' + time_str + '\n' + speed_str + '\n' + '-'*80
print '\033[2J' + display_str
elif (i+1)%seqs_per_time_update == 0 and time_updates:
c_time = time.time() - start_time
eta = ((len(seqs) - (i+1))/float(i+1))*c_time
if c_time > 86400: #more than a day
c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 3600: #more than an hr
c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 60: #more than a min
c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
else:
c_time_str = '%.2f seconds.'%(c_time)
if eta > 86400: #more than a day
eta_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(eta)/86400, (int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
elif eta > 3600: #more than an hr
eta_str = '%d hours, %d minutes, and %.2f seconds.'%((int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
elif eta > 60: #more than a min
eta_str = '%d minutes and %.2f seconds.'%((int(eta)/60)%60, eta%60)
else:
eta_str = '%.2f seconds.'%(eta)
print 'Pgen computed for %d sequences in: %s Estimated time remaining: %s'%(i+1, c_time_str, eta_str)
c_time = time.time() - start_time
if c_time > 86400: #more than a day
c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 3600: #more than an hr
c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
elif c_time > 60: #more than a min
c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
else:
c_time_str = '%.2f seconds.'%(c_time)
print 'Completed Pgen computation for %d sequences: in %s'%(len(seqs), c_time_str)
outfile.close()
else: #NO OUTFILE -- print directly to stdout
start_time = time.time()
for i, seq in enumerate(seqs):
if seq_types[i] == 'aaseq':
#Compute Pgen and print out
c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
if seq_types[i] == 'regex':
#Compute Pgen and print out
c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_types[i] == 'ntseq':
ntseq = seq
if len(ntseq) % 3 == 0: #inframe sequence
aaseq = nt2aa(ntseq)
#Compute Pgen and print out based on recomb_type and seq_type_out
if seq_type_out is None:
c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_type_out == 'ntseq':
c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
elif seq_type_out == 'aaseq':
c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
if seq_type_out is None:
c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
elif seq_type_out == 'ntseq':
c_pgen_line = ntseq + delimiter_out + '0'
elif seq_type_out == 'aaseq':
c_pgen_line = 'out_of_frame' + delimiter_out + '0'
print c_pgen_line
def evaluate(self,
seqs,
num_threads,
use_allele=True,
default_allele=None):
"""Evaluate a given nucleotide CDR3 sequences using OLGA.
This function also checks if the given input sequence file contains the gene index columns for the V and J gene.
If so, then the V and J gene masks in these columns are used to increase calculation accuracy of the generation
probabality values.
Parameters
----------
seqs : pandas.DataFrame
A pandas dataframe object containing a column with nucleotide CDR3 sequences and/or amino acid sequences.
num_threads : int
The number of threads to use when processing the sequences.
use_allele : bool, optional
If True, the allele information from the input genes is used instead of the 'default_allele' value (default: True).
default_allele : str, optional
A default allele value to use when spliting gene choices, and 'use_allele' option is False (default: None).
Returns
-------
pandas.DataFrame
Containing the sequence index number, the generation probability of nucleotide sequence if given and the
generation probability of aminoacid sequence if given.
Raises
------
TypeError
When the model type does not equal 'VDJ' or 'VJ'.
"""
# Set the evaluation objects.
pgen_model = None
if self.igor_model.get_type() == "VDJ":
pgen_model = olga_pgen.GenerationProbabilityVDJ(
self.igor_model.get_generative_model(),
self.igor_model.get_genomic_data())
elif self.igor_model.get_type() == "VJ":
pgen_model = olga_pgen.GenerationProbabilityVJ(
self.igor_model.get_generative_model(),
self.igor_model.get_genomic_data())
else:
raise TypeError(
"OLGA could not create a GenerationProbability object since model is not of type 'VDJ' or 'VJ'"
)
# Insert amino acid sequence column if not existent.
if (self.col_names['NT_COL'] in seqs.columns
and not self.col_names['AA_COL'] in seqs.columns):
seqs.insert(
seqs.columns.get_loc(self.col_names['NT_COL']) + 1,
self.col_names['AA_COL'], numpy.nan)
seqs[self.col_names['AA_COL']] = seqs[self.col_names['NT_COL']] \
.apply(nucleotides_to_aminoacids)
# Use multiprocessing to evaluate the sequences in chunks and return.
result = multiprocess_array(ary=seqs,
func=self._evaluate,
num_workers=num_threads,
model=pgen_model,
use_allele=use_allele,
default_allele=default_allele)
result = pandas.concat(result, axis=0, copy=False)
return result
