def personal_genotype(sample_name,
genotype_file,
chain,
gene_order_file,
html=True):
output_path = make_output_file('html' if html else 'pdf')
file_type = 'T' if html else 'F'
cmd_line = [
"-i", genotype_file, "-o", output_path, "-t", file_type, "--samp",
sample_name, "-g", gene_order_file, "-c", chain
]
if run_rscript(MULTIPLE_GENOTYPE_SCRIPT,
cmd_line) and os.path.getsize(output_path) > 0:
return output_path
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if len(rep_samples) == 0:
raise BadRequest('No repertoire-derived genotypes were selected.')
if format not in (['pdf', 'html']):
raise BadRequest('Invalid format requested')
single_sample_filter = 1 if params[
'single_sample'] == 'One Selected Sample' else 0
calc_by_clone = 1 if params['calculate_by'] == 'Number of Clones' else 0
chain, samples_by_dataset = collate_samples(rep_samples)
# Format we need to produce is [(gene_name, hetero count, h**o count),...]
genes_frequencies = defaultdict(list)
for dataset in samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
for sample_chunk in chunk_list(samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(Sample.sample_name, Sample.genotype, Sample.patient_id)\
.filter(Sample.sample_name.in_(sample_chunk))\
.filter(Sample.sample_group >= single_sample_filter)\
.all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
i = 0
sample_list_len = len(sample_list)
frequencies = session.query(GenesDistribution.sample_id, Gene.name, GenesDistribution.frequency)\
.join(Gene)\
.join(Sample)\
.filter(GenesDistribution.count_by_clones == calc_by_clone)\
.filter(Gene.name.in_(wanted_genes)) \
.filter(Sample.sample_name.in_(sample_list)) \
.all()
for frequency in frequencies:
genes_frequencies[frequency[1]].append(
round(float(frequency[2]), 2))
labels = ['GENE', 'FREQ']
genes_frequencies_df = pd.DataFrame(columns=labels)
for gene, usages in genes_frequencies.items():
genes_frequencies_df = genes_frequencies_df.append(
{
'GENE': gene,
'FREQ': ",".join([str(x) for x in usages])
},
ignore_index=True)
input_path = make_output_file('tab')
genes_frequencies_df.to_csv(input_path, sep='\t', index=False)
output_path = make_output_file(format)
attachment_filename = '%s_gene_frequency.pdf' % species
locus_order = ('sort_order' in params and params['sort_order'] == 'Locus')
gene_order_file = get_multiple_order_file(species,
samples_by_dataset.keys(),
locus_order=locus_order)
cmd_line = [
"-i", input_path, "-o", output_path, "-t",
'T' if format == 'html' else 'F', "-c", chain, "-g", gene_order_file
]
if run_rscript(GENE_FREQUENCY_PLOT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format, attachment_filename)
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if format != 'pdf' and format != 'xls':
raise BadRequest('Invalid format requested')
rep_samples_by_dataset = {}
for rep_sample in rep_samples:
if rep_sample['dataset'] not in rep_samples_by_dataset:
rep_samples_by_dataset[rep_sample['dataset']] = []
rep_samples_by_dataset[rep_sample['dataset']].append(
rep_sample['sample_name'])
# Format we need to produce is [gene_name, [allele names], [allele appearances], gene appearances]
# Start with a dict indexed by gene, then convert to appropriately sorted list
counts = {}
for dataset in rep_samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
appearances = []
for sample_chunk in chunk_list(rep_samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(
Sample.sample_name, Sample.genotype, Sample.patient_id).filter(
Sample.sample_name.in_(sample_chunk)).all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
app_query = session.query(AllelesSample.patient_id, Patient.patient_name, Gene.name, Allele.name, Sample.sample_name, Gene.locus_order, Gene.alpha_order)\
.filter(Sample.id == AllelesSample.sample_id)\
.filter(Allele.id == AllelesSample.allele_id)\
.filter(Gene.id == Allele.gene_id)\
.filter(Patient.id == AllelesSample.patient_id)\
.filter(Sample.sample_name.in_(sample_list))\
.filter(Gene.name.in_(wanted_genes))
if params['novel_alleles'] == 'Exclude':
app_query = app_query.filter(Allele.novel == 0)
if params['ambiguous_alleles'] == 'Exclude':
app_query = app_query.filter(Allele.is_single_allele == 1)
appearances.extend(app_query.all())
for app in appearances:
_, patient_name, gene, allele, sample, locus_order, alpha_order = app
allele = allele.split('*', 1)[1].upper()
if gene not in counts:
if params['sort_order'] == 'Alphabetic':
counts[gene] = [{}, [], alpha_order]
else:
counts[gene] = [{}, [], locus_order]
if allele not in counts[gene][0]:
counts[gene][0][allele] = []
if patient_name not in counts[gene][0][allele]:
counts[gene][0][allele].append(patient_name)
if patient_name not in counts[gene][1]:
counts[gene][1].append(patient_name)
gen_samples_by_dataset = {}
for gen_sample in genomic_samples:
if gen_sample['dataset'] not in gen_samples_by_dataset:
gen_samples_by_dataset[gen_sample['dataset']] = []
gen_samples_by_dataset[gen_sample['dataset']].append(
gen_sample['identifier'])
for dataset in gen_samples_by_dataset.keys():
session = genomic_dbs[species][dataset].session
appearances = []
for sample_chunk in chunk_list(gen_samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(GenomicSubject.identifier).filter(
GenomicSubject.identifier.in_(sample_chunk)).all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
app_query = session.query(GenomicSubject.identifier,
GenomicGene.name,
GenomicSequence.name,
GenomicGene.locus_order,
GenomicGene.alpha_order)\
.filter(GenomicSubject.id == GenomicSubjectSequence.subject_id) \
.filter(GenomicSequence.id == GenomicSubjectSequence.sequence_id) \
.filter(GenomicGene.id == GenomicSequence.gene_id) \
.filter(GenomicSequence.type.in_(['V-REGION', 'D-REGION', 'J-REGION'])) \
.filter(GenomicSubject.identifier.in_(sample_list))\
.filter(GenomicGene.name.in_(wanted_genes))
if params['novel_alleles'] == 'Exclude':
app_query = app_query.filter(GenomicSequence.novel == 0)
appearances.extend(app_query.all())
for app in appearances:
patient_name, gene, allele, locus_order, alpha_order = app
allele = allele.split('*', 1)[1].upper()
if gene not in counts:
if params['sort_order'] == 'Alphabetic':
counts[gene] = [{}, [], alpha_order]
else:
counts[gene] = [{}, [], locus_order]
if allele not in counts[gene][0]:
counts[gene][0][allele] = []
if patient_name not in counts[gene][0][allele]:
counts[gene][0][allele].append(patient_name)
if patient_name not in counts[gene][1]:
counts[gene][1].append(patient_name)
single_alleles = []
multi_alleles = []
for gene, (alleles, total, order) in counts.items():
row = [
gene,
sorted(list(alleles.keys())),
[len(alleles[a]) for a in sorted(alleles.keys())],
len(total), order
]
if len(alleles) > 1:
multi_alleles.append(row)
else:
single_alleles.append(row)
multi_alleles.sort(key=lambda row: row[4])
multi_alleles = [m[:4] for m in multi_alleles]
single_alleles.sort(key=lambda row: row[4])
s = ['Single allele genes', [], []]
for (gene, alleles, counts, _, _) in single_alleles:
s[1].append(gene + '\n' + alleles[0])
s[2].append(counts[0])
multi_alleles.append(s)
input_path = make_output_file('xls')
output_path = make_output_file('pdf')
book = xlwt.Workbook()
for row in multi_alleles:
if len(row[1]) > 0:
write_gene(book, row)
book.save(input_path)
if format == 'xls':
return send_report(input_path, format,
'%s_allele_appearance.xls' % species)
cmd_line = ["-i", input_path, "-o", output_path]
if run_rscript(APPEARANCE_SCRIPT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format,
'%s_allele_appearance.pdf' % species)
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if len(rep_samples) == 0:
raise BadRequest('No repertoire-derived genotypes were selected.')
if format not in ['pdf', 'html']:
raise BadRequest('Invalid format requested')
html = (format == 'html')
chain, samples_by_dataset = collate_samples(rep_samples)
genotypes = pd.DataFrame()
for dataset in samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
primer_trans, gene_subs = find_primer_translations(session)
sample_list = []
for sample_chunk in chunk_list(samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list.extend(
session.query(Sample.sample_name, Sample.genotype,
Sample.patient_id).filter(
Sample.sample_name.in_(sample_chunk)).all())
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
if len(wanted_genes) > 0:
for (name, genotype, patient_id) in sample_list:
sample_path = os.path.join(VDJBASE_SAMPLE_PATH, species,
dataset,
genotype.replace('samples/', ''))
if not os.path.isfile(sample_path):
continue
genotype = pd.read_csv(sample_path, sep='\t', dtype=str)
genotype = trans_df(genotype)
# translate pipeline allele names to VDJbase allele names
for col in ['alleles', 'GENOTYPED_ALLELES']:
genotype[col] = [
translate_primer_alleles(x, y, primer_trans)
for x, y in zip(genotype['gene'], genotype[col])
]
genotype['gene'] = [
translate_primer_genes(x, gene_subs)
for x in genotype['gene']
]
genotype = genotype[genotype.gene.isin(wanted_genes)]
subject_name = name if len(
samples_by_dataset) == 1 else dataset + '_' + name
if 'subject' not in genotype.columns.values:
genotype.insert(0, 'subject', subject_name)
else:
genotype.subject = subject_name
genotypes = genotypes.append(genotype)[
genotype.columns.tolist()]
if len(genotypes) == 0:
raise BadRequest('No records matching the filter criteria were found.')
geno_path = make_output_file('csv')
genotypes.to_csv(geno_path, sep='\t')
if format == 'pdf':
attachment_filename = '%s_genotype.pdf' % species
else:
attachment_filename = None
locus_order = ('sort_order' in params and params['sort_order'] == 'Locus')
gene_order_file = get_multiple_order_file(species,
samples_by_dataset.keys(),
locus_order=locus_order)
output_path = make_output_file('html' if html else 'pdf')
file_type = 'T' if html else 'F'
cmd_line = [
"-i", geno_path, "-o", output_path, "-t", file_type, "-k",
str(params['f_kdiff']), "-c", chain, "-g", gene_order_file
]
if run_rscript(HEATMAP_GENOTYPE_SCRIPT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format, attachment_filename)
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if len(rep_samples) == 0:
raise BadRequest('No repertoire-derived genotypes were selected.')
if format != 'pdf':
raise BadRequest('Invalid format requested')
html = (format == 'html')
chain, samples_by_dataset = collate_samples(rep_samples)
haplotypes = pd.DataFrame()
for dataset in samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
primer_trans, gene_subs = find_primer_translations(session)
haplos = []
for sample_chunk in chunk_list(samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(
Sample.sample_name, Sample.genotype, Sample.patient_id).filter(
Sample.sample_name.in_(sample_chunk)).all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
haplo_query = session.query(Sample.sample_name, HaplotypesFile.file)\
.filter(Sample.sample_name.in_(sample_list))\
.join(SamplesHaplotype, Sample.id == SamplesHaplotype.samples_id)\
.filter(SamplesHaplotype.haplotypes_file_id == HaplotypesFile.id)\
.filter(HaplotypesFile.by_gene == params['haplo_gene'])
haplos.extend(haplo_query.all())
for name, filename in haplos:
sample_path = os.path.join(VDJBASE_SAMPLE_PATH, species, dataset,
filename.replace('samples/', ''))
if not os.path.isfile(sample_path):
raise BadRequest('Haplotype file %s is missing.' %
(sample_path))
haplotype = pd.read_csv(sample_path, sep='\t', dtype=str)
haplotype = trans_df(haplotype)
haplotype['subject'] = name if len(
samples_by_dataset) == 1 else dataset + '_' + name
# translate pipeline allele names to VDJbase allele names
col_names = list(haplotype.columns.values)
for i in (2, 3, 4):
haplotype[col_names[i]] = [
translate_primer_alleles(x, y, primer_trans)
for x, y in zip(haplotype['gene'], haplotype[col_names[i]])
]
haplotype['gene'] = [
translate_primer_genes(x, gene_subs) for x in haplotype['gene']
]
haplotype = haplotype[haplotype.gene.isin(wanted_genes)]
haplotypes = pd.concat(
[haplotypes, haplotype], keys=None,
ignore_index=True)[haplotype.columns.tolist()]
if len(haplotypes) == 0:
raise BadRequest('No records matching the filter criteria were found.')
haplo_path = make_output_file('tsv')
haplotypes.to_csv(haplo_path, sep='\t', index=False)
attachment_filename = '%s_haplotype_heatmap.pdf' % species
if not params['f_kdiff'] or params['f_kdiff'] == '':
params['f_kdiff'] = 0
locus_order = ('sort_order' in params and params['sort_order'] == 'Locus')
gene_order_file = get_multiple_order_file(species,
samples_by_dataset.keys(),
locus_order=locus_order)
output_path = make_output_file('html' if html else 'pdf')
cmd_line = [
"-i", haplo_path, "-o", output_path, "-k",
str(params['f_kdiff']), "-c", chain, "-g", gene_order_file
]
if run_rscript(HEATMAP_HAPLOTYPE_SCRIPT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format, attachment_filename)
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if len(rep_samples) == 0:
raise BadRequest('No repertoire-derived genotypes were selected.')
if format != 'html':
raise BadRequest('Invalid format requested')
kdiff = float(params['f_kdiff']
) if 'f_kdiff' in params and params['f_kdiff'] != '' else 0
chain, samples_by_dataset = collate_samples(rep_samples)
if len(samples_by_dataset
) > 1 and params['ambiguous_alleles'] != 'Exclude':
raise BadRequest(
'Ambiguous alleles cannot be processed across multiple datasets')
# Format we need to produce is [(gene_name, hetero count, h**o count),...]
gene_allele_counts = {}
for dataset in samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
allele_recs = []
for sample_chunk in chunk_list(samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(
Sample.sample_name, Sample.genotype, Sample.patient_id).filter(
Sample.sample_name.in_(sample_chunk)).all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
query = session.query(Gene.name, Allele.id, Gene.type) \
.join(Allele) \
.join(AllelesSample) \
.join(Sample) \
.join(Patient, Patient.id == Sample.patient_id) \
.filter(Gene.name.in_(wanted_genes)) \
.filter(Allele.name.notlike('%Del%')) \
.filter(Allele.name.notlike('%OR%')) \
.filter(Sample.sample_name.in_(sample_list)) \
.filter(AllelesSample.kdiff >= kdiff)
if 'sort_order' in params and params['sort_order'] == 'Locus':
query = query.order_by(Gene.locus_order, Patient.id, Allele.id)
else:
query = query.order_by(Gene.alpha_order, Patient.id, Allele.id)
if params['novel_alleles'] == 'Exclude':
query = query.filter(Allele.novel == 0)
if params['ambiguous_alleles'] == 'Exclude':
query = query.filter(Allele.is_single_allele == 1)
allele_recs.extend(query.all())
i = 0
while i < len(allele_recs):
(gene_name, allele_id, gene_type) = allele_recs[i]
gene_allele_ids = []
while i < len(allele_recs):
if allele_recs[i][0] != gene_name:
break
allele_id = allele_recs[i][1]
gene_allele_ids.append(allele_id)
i += 1
gene_allele_ids = set(gene_allele_ids)
# If we have both an unambiguous allele and an ambiguous allele containing that unambiguous one,
# drop the unambiguous one because it is already counted
if (params['ambiguous_alleles'] != 'Exclude'):
patterns = session.query(AllelesPattern.pattern_id)\
.filter(AllelesPattern.allele_in_p_id.in_(gene_allele_ids))\
.filter(AllelesPattern.pattern_id.in_(gene_allele_ids))\
.all()
if patterns is not None and len(patterns) > 0:
patterns = set([pattern[0] for pattern in patterns])
gene_allele_ids = gene_allele_ids - patterns
if gene_name not in gene_allele_counts:
gene_allele_counts[gene_name] = gene_allele_ids
else:
gene_allele_counts[gene_name] |= gene_allele_ids
listed_allele_count = []
for gene, alleles in gene_allele_counts.items():
listed_allele_count.append((gene, len(alleles)))
labels = ['GENE', 'COUNT']
input_path = make_output_file('tab')
df = pd.DataFrame(listed_allele_count, columns=labels)
df.to_csv(input_path, sep='\t', index=False)
output_path = make_output_file('html')
cmd_line = ["-i", input_path, "-o", output_path, "-c", chain]
if run_rscript(ALLELE_USAGE_SCRIPT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format)
else:
raise BadRequest('No output from report')
def run(format, species, genomic_datasets, genomic_samples, rep_datasets,
rep_samples, params):
if len(rep_samples) == 0:
raise BadRequest('No repertoire-derived genotypes were selected.')
if format != 'html':
raise BadRequest('Invalid format requested')
kdiff = float(params['f_kdiff']
) if 'f_kdiff' in params and params['f_kdiff'] != '' else 0
chain, samples_by_dataset = collate_samples(rep_samples)
# Format we need to produce is [(gene_name, hetero count, h**o count),...]
gene_hetrozygous_dis = {}
for dataset in samples_by_dataset.keys():
session = vdjbase_dbs[species][dataset].session
allele_sample_recs = []
for sample_chunk in chunk_list(samples_by_dataset[dataset],
SAMPLE_CHUNKS):
sample_list = session.query(
Sample.sample_name, Sample.genotype, Sample.patient_id).filter(
Sample.sample_name.in_(sample_chunk)).all()
sample_list, wanted_genes = apply_rep_filter_params(
params, sample_list, session)
sample_list = [s[0] for s in sample_list]
query = session.query(Gene.name, Patient.id, Allele.id, Sample.sample_name, Gene.locus_order, AllelesSample.kdiff, Allele.name) \
.join(Allele, Gene.id == Allele.gene_id) \
.join(AllelesSample, Allele.id == AllelesSample.allele_id) \
.join(Sample, Sample.id == AllelesSample.sample_id) \
.join(Patient, Patient.id == Sample.patient_id) \
.filter(Gene.name.in_(wanted_genes)) \
.filter(Allele.name.notlike('%Del%')) \
.filter(Allele.name.notlike('%OR%')) \
.filter(Sample.sample_name.in_(sample_list)) \
.filter(AllelesSample.kdiff >= kdiff)
if 'sort_order' in params and params['sort_order'] == 'Locus':
query = query.order_by(Gene.locus_order, Patient.id, Allele.id)
else:
query = query.order_by(Gene.alpha_order, Patient.id, Allele.id)
if params['ambiguous_alleles'] == 'Exclude':
query = query.filter(Allele.is_single_allele == True)
allele_sample_recs.extend(query.all())
# As the result is indexed, run over each gene in turn, count the number of alleles found in each patient, update h_counts accordingly
i = 0
target_gene = ''
while i < len(allele_sample_recs):
target_gene = allele_sample_recs[i][0]
h_counts = [0, 0]
while i < len(allele_sample_recs):
if allele_sample_recs[i][0] != target_gene:
break
target_patient = allele_sample_recs[i][1]
patient_allele_ids = []
while i < len(allele_sample_recs):
if allele_sample_recs[i][
0] != target_gene or allele_sample_recs[i][
1] != target_patient:
break
patient_allele_ids.append(allele_sample_recs[i][2])
i += 1
patient_allele_ids = set(patient_allele_ids)
# If we have both an unambiguous allele and an ambiguous allele containing that unambiguous one,
# drop the unambiguous one because it is already counted
if params['ambiguous_alleles'] != 'Exclude':
patterns = session.query(AllelesPattern.pattern_id)\
.filter(AllelesPattern.allele_in_p_id.in_(patient_allele_ids))\
.filter(AllelesPattern.pattern_id.in_(patient_allele_ids))\
.all()
if patterns is not None and len(patterns) > 0:
patterns = set([pattern[0] for pattern in patterns])
patient_allele_ids = patient_allele_ids - patterns
if len(patient_allele_ids) > 1:
h_counts[1] += 1
elif len(patient_allele_ids) > 0:
h_counts[0] += 1
if target_gene not in gene_hetrozygous_dis:
gene_hetrozygous_dis[target_gene] = (target_gene, h_counts[0],
h_counts[1])
else:
gene_hetrozygous_dis[target_gene] = (
target_gene,
gene_hetrozygous_dis[target_gene][1] + h_counts[0],
gene_hetrozygous_dis[target_gene][2] + h_counts[1])
haplo_path = make_output_file('tab')
labels = ['GENE', 'HM', 'HT']
df = pd.DataFrame(gene_hetrozygous_dis.values(), columns=labels)
df.to_csv(haplo_path, sep='\t', index=False)
output_path = make_output_file('html')
cmd_line = [
"-i",
haplo_path,
"-o",
output_path,
"-c",
chain,
]
if run_rscript(HETEROZYGOSITY_SCRIPT, cmd_line) and os.path.isfile(
output_path) and os.path.getsize(output_path) != 0:
return send_report(output_path, format)
else:
raise BadRequest('No output from report')