"""

Connects to the specific database.

"""

client = pymongo.MongoClient(host=app.config['DB_HOST'], port=app.config['DB_PORT'])

"""

Returns a generator of parsed record objects (as returned by record_parser) for the i'th out n subset of records

across all the given tabix_file(s). The records are split by files and contigs within files, with 1/n of all contigs

from all files being assigned to this the i'th subset.

Args:

tabix_filenames: a list of one or more tabix-indexed files. These will be opened using pysam.Tabixfile

subset_i: zero-based number

subset_n: total number of subsets

record_parser: a function that takes a file-like object and returns a generator of parsed records

"""

start_time = time.time()

open_tabix_files = [pysam.Tabixfile(tabix_filename) for tabix_filename in tabix_filenames]

tabix_file_contig_pairs = [(tabix_file, contig) for tabix_file in open_tabix_files for contig in tabix_file.contigs]

tabix_file_contig_subset = tabix_file_contig_pairs[subset_i : : subset_n] # get every n'th tabix_file/contig pair

short_filenames = ", ".join(map(os.path.basename, tabix_filenames))

num_file_contig_pairs = len(tabix_file_contig_subset)

print(("Loading subset %(subset_i)s of %(subset_n)s total: %(num_file_contig_pairs)s contigs from "

"%(short_filenames)s") % locals())

counter = 0

for tabix_file, contig in tabix_file_contig_subset:

header_iterator = tabix_file.header

records_iterator = tabix_file.fetch(contig, 0, 10**9, multiple_iterators=True)

for parsed_record in record_parser(itertools.chain(header_iterator, records_iterator)):

counter += 1

yield parsed_record

if counter % 100000 == 0:

seconds_elapsed = int(time.time()-start_time)

print(("Loaded %(counter)s records from subset %(subset_i)s of %(subset_n)s from %(short_filenames)s "

"(%(seconds_elapsed)s seconds)") % locals())

def load_coverage(coverage_files, i, n, db):

coverage_generator = parse_tabix_file_subset(coverage_files, i, n, get_base_coverage_from_file)

try:

db.base_coverage.insert(coverage_generator, w=0)

except pymongo.errors.InvalidOperation:

pass # handle error when coverage_generator is empty

db = get_db()

db.base_coverage.drop()

print("Dropped db.base_coverage")

# load coverage first; variant info will depend on coverage

start_time = time.time()

db.base_coverage.ensure_index('xpos')

procs = []

coverage_files = app.config['BASE_COVERAGE_FILES']

num_procs = app.config['LOAD_DB_PARALLEL_PROCESSES']

random.shuffle(app.config['BASE_COVERAGE_FILES'])

for i in range(num_procs):

p = Process(target=load_coverage, args=(coverage_files, i, num_procs, db))

p.start()

procs.append(p)

return procs

def load_variants(sites_file, i, n, db):

variants_generator = parse_tabix_file_subset([sites_file], i, n, get_variants_from_sites_vcf)

try:

db.variants.insert(variants_generator, w=0)

except pymongo.errors.InvalidOperation:

pass # handle error when variant_generator is empty

db = get_db()

db.variants.drop()

print("Dropped db.variants")

# grab variants from sites VCF

start_time = time.time()

db.variants.ensure_index('xpos')

db.variants.ensure_index('xstart')

db.variants.ensure_index('xstop')

db.variants.ensure_index('rsid')

db.variants.ensure_index('genes')

db.variants.ensure_index('transcripts')

sites_vcfs = app.config['SITES_VCFS']

if len(sites_vcfs) > 1:

raise Exception("More than one sites vcf file found: %s" % sites_vcfs)

procs = []

num_procs = app.config['LOAD_DB_PARALLEL_PROCESSES']

for i in range(num_procs):

p = Process(target=load_variants, args=(sites_vcfs[0], i, num_procs, db))

p.start()

procs.append(p)

return procs

db = get_db()

db.genes.drop()

db.transcripts.drop()

db.exons.drop()

print 'Dropped db.genes, db.transcripts, and db.exons.'

start_time = time.time()

#size = os.path.getsize(app.config['CANONICAL_TRANSCRIPT_FILE'])

canonical_transcripts = {}

with gzip.open(app.config['CANONICAL_TRANSCRIPT_FILE']) as canonical_transcript_file:

#progress = xbrowse.utils.get_progressbar(size, 'Loading Canonical Transcripts')

for gene, transcript in get_canonical_transcripts(canonical_transcript_file):

canonical_transcripts[gene] = transcript

#progress.update(canonical_transcript_file.fileobj.tell())

#progress.finish()

#size = os.path.getsize(app.config['OMIM_FILE'])

omim_annotations = {}

with gzip.open(app.config['OMIM_FILE']) as omim_file:

#progress = xbrowse.utils.get_progressbar(size, 'Loading OMIM accessions')

for fields in get_omim_associations(omim_file):

if fields is None:

continue

gene, transcript, accession, description = fields

omim_annotations[gene] = (accession, description)

#progress.update(omim_file.fileobj.tell())

#progress.finish()

#size = os.path.getsize(app.config['DBNSFP_FILE'])

dbnsfp_info = {}

with gzip.open(app.config['DBNSFP_FILE']) as dbnsfp_file:

#progress = xbrowse.utils.get_progressbar(size, 'Loading dbNSFP info')

for dbnsfp_gene in get_dbnsfp_info(dbnsfp_file):

other_names = [other_name.upper() for other_name in dbnsfp_gene['gene_other_names']]

dbnsfp_info[dbnsfp_gene['ensembl_gene']] = (dbnsfp_gene['gene_full_name'], other_names)

#progress.update(dbnsfp_file.fileobj.tell())

#progress.finish()

print 'Done loading metadata. Took %s seconds' % int(time.time() - start_time)

# grab genes from GTF

start_time = time.time()

#size = os.path.getsize(app.config['GENCODE_GTF'])

#progress = xbrowse.utils.get_progressbar(size, 'Loading Genes')

with gzip.open(app.config['GENCODE_GTF']) as gtf_file:

for gene in get_genes_from_gencode_gtf(gtf_file):

gene_id = gene['gene_id']

if gene_id in canonical_transcripts:

gene['canonical_transcript'] = canonical_transcripts[gene_id]

if gene_id in omim_annotations:

gene['omim_accession'] = omim_annotations[gene_id][0]

gene['omim_description'] = omim_annotations[gene_id][1]

if gene_id in dbnsfp_info:

gene['full_gene_name'] = dbnsfp_info[gene_id][0]

gene['other_names'] = dbnsfp_info[gene_id][1]

db.genes.insert(gene, w=0)

#progress.update(gtf_file.fileobj.tell())

#progress.finish()

print 'Done loading genes. Took %s seconds' % int(time.time() - start_time)

start_time = time.time()

db.genes.ensure_index('gene_id')

db.genes.ensure_index('gene_name_upper')

db.genes.ensure_index('gene_name')

db.genes.ensure_index('other_names')

db.genes.ensure_index('xstart')

db.genes.ensure_index('xstop')

print 'Done indexing gene table. Took %s seconds' % int(time.time() - start_time)

# and now transcripts

start_time = time.time()

#size = os.path.getsize(app.config['GENCODE_GTF'])

#progress = xbrowse.utils.get_progressbar(size, 'Loading Transcripts')

with gzip.open(app.config['GENCODE_GTF']) as gtf_file:

db.transcripts.insert((transcript for transcript in get_transcripts_from_gencode_gtf(gtf_file)), w=0)

#progress.update(gtf_file.fileobj.tell())

#progress.finish()

print 'Done loading transcripts. Took %s seconds' % int(time.time() - start_time)

start_time = time.time()

db.transcripts.ensure_index('transcript_id')

db.transcripts.ensure_index('gene_id')

print 'Done indexing transcript table. Took %s seconds' % int(time.time() - start_time)

# Building up gene definitions

start_time = time.time()

#size = os.path.getsize(app.config['GENCODE_GTF'])

#progress = xbrowse.utils.get_progressbar(size, 'Loading Exons')

with gzip.open(app.config['GENCODE_GTF']) as gtf_file:

db.exons.insert((exon for exon in get_exons_from_gencode_gtf(gtf_file)), w=0)

#progress.update(gtf_file.fileobj.tell())

#progress.finish()

print 'Done loading exons. Took %s seconds' % int(time.time() - start_time)

start_time = time.time()

db.exons.ensure_index('exon_id')

db.exons.ensure_index('transcript_id')

db.exons.ensure_index('gene_id')

print 'Done indexing exon table. Took %s seconds' % int(time.time() - start_time)

db = get_db()

def load_dbsnp(dbsnp_file, i, n, db):

if os.path.isfile(dbsnp_file + ".tbi"):

dbsnp_record_generator = parse_tabix_file_subset([dbsnp_file], i, n, get_snp_from_dbsnp_file)

try:

db.dbsnp.insert(dbsnp_record_generator, w=0)

except pymongo.errors.InvalidOperation:

pass # handle error when coverage_generator is empty

else:

with gzip.open(dbsnp_file) as f:

db.dbsnp.insert((snp for snp in get_snp_from_dbsnp_file(f)), w=0)

db.dbsnp.drop()

db.dbsnp.ensure_index('rsid')

db.dbsnp.ensure_index('xpos')

start_time = time.time()

dbsnp_file = app.config['DBSNP_FILE']

print "Loading dbsnp from %s" % dbsnp_file

if os.path.isfile(dbsnp_file + ".tbi"):

num_procs = app.config['LOAD_DB_PARALLEL_PROCESSES']

else:

# see if non-tabixed .gz version exists

if os.path.isfile(dbsnp_file):

print(("WARNING: %(dbsnp_file)s.tbi index file not found. Will use single thread to load dbsnp."

"To create a tabix-indexed dbsnp file based on UCSC dbsnp, do: \n"

" wget http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/snp141.txt.gz \n"

" gzcat snp141.txt.gz | cut -f 1-5 | bgzip -c > snp141.txt.bgz \n"

" tabix -0 -s 2 -b 3 -e 4 snp141.txt.bgz") % locals())

num_procs = 1

else:

raise Exception("dbsnp file %s(dbsnp_file)s not found." % locals())

procs = []

for i in range(num_procs):

p = Process(target=load_dbsnp, args=(dbsnp_file, i, num_procs, db))

p.start()

procs.append(p)

return procs

#print 'Done loading dbSNP. Took %s seconds' % int(time.time() - start_time)

#start_time = time.time()

#db.dbsnp.ensure_index('rsid')

"""

Load the database

"""

# Initialize database

# Don't need to explicitly create tables with mongo, just indices

all_procs = []

for load_function in [load_variants_file, load_dbsnp_file, load_base_coverage, load_gene_models]:

procs = load_function()

all_procs.extend(procs)

print("Started %s processes to run %s" % (len(procs), load_function.__name__))

"""

This is essentially a compile step that generates all cached resources.

Creates files like autocomplete_entries.txt

Should be run on every redeploy.

"""

# create autocomplete_entries.txt

autocomplete_strings = []

for gene in get_db().genes.find():

autocomplete_strings.append(gene['gene_name'])

if 'other_names' in gene:

autocomplete_strings.extend(gene['other_names'])

f = open(os.path.join(os.path.dirname(__file__), 'autocomplete_strings.txt'), 'w')

for s in sorted(autocomplete_strings):

f.write(s+'\n')

f.close()

# create static gene pages for genes in

if not os.path.exists(GENE_CACHE_DIR):

os.makedirs(GENE_CACHE_DIR)

# get list of genes ordered by num_variants

for gene_id in GENES_TO_CACHE:

try:

page_content = get_gene_page_content(gene_id)

except Exception:

print Exception

continue

f = open(os.path.join(GENE_CACHE_DIR, '{}.html'.format(gene_id)), 'w')

f.write(page_content)

import numpy

db = get_db()

print 'Reading %s variants...' % db.variants.count()

metrics = defaultdict(list)

binned_metrics = defaultdict(list)

progress = 0

start_time = time.time()

for variant in db.variants.find(fields=['quality_metrics', 'site_quality', 'allele_num', 'allele_count']):

for metric, value in variant['quality_metrics'].iteritems():

metrics[metric].append(float(value))

qual = float(variant['site_quality'])

metrics['site_quality'].append(qual)

if variant['allele_num'] == 0: continue

if variant['allele_count'] == 1:

binned_metrics['singleton'].append(qual)

elif variant['allele_count'] == 2:

binned_metrics['doubleton'].append(qual)

else:

for af in AF_BUCKETS:

if float(variant['allele_count'])/variant['allele_num'] < af:

binned_metrics[af].append(qual)

break

progress += 1

if not progress % 100000:

print 'Read %s variants. Took %s seconds' % (progress, int(time.time() - start_time))

print 'Done reading variants. Dropping metrics database... '

db.metrics.drop()

print 'Dropped metrics database. Calculating metrics...'

for metric in metrics:

hist = numpy.histogram(metrics[metric], bins=40)

edges = hist[1]

# mids = [(edges[i]+edges[i+1])/2 for i in range(len(edges)-1)]

lefts = [edges[i] for i in range(len(edges)-1)]

db.metrics.insert({

'metric': metric,

'mids': lefts,

'hist': list(hist[0])

})

for metric in binned_metrics:

hist = numpy.histogram(map(numpy.log, binned_metrics[metric]), bins=40)

edges = hist[1]

mids = [(edges[i]+edges[i+1])/2 for i in range(len(edges)-1)]

db.metrics.insert({

'metric': 'binned_%s' % metric,

'mids': mids,

'hist': list(hist[0])

})

db.metrics.ensure_index('metric')

"""

Opens a new database connection if there is none yet for the

current application context.

"""

if not hasattr(g, 'db_conn'):

g.db_conn = connect_db()

if not hasattr(g, 'autocomplete_strings'):

g.autocomplete_strings = [s.strip() for s in open(os.path.join(os.path.dirname(__file__), 'autocomplete_strings.txt'))]

suggestions = lookups.get_awesomebar_suggestions(g, query)

db = get_db()

query = request.args.get('query')

datatype, identifier = lookups.get_awesomebar_result(db, query)

print "Searched for %s: %s" % (datatype, identifier)

if datatype == 'gene':

return redirect('/gene/{}'.format(identifier))

elif datatype == 'transcript':

return redirect('/transcript/{}'.format(identifier))

elif datatype == 'variant':

return redirect('/variant/{}'.format(identifier))

elif datatype == 'region':

return redirect('/region/{}'.format(identifier))

elif datatype == 'dbsnp_variant_set':

return redirect('/dbsnp/{}'.format(identifier))

elif datatype == 'error':

return redirect('/error/{}'.format(identifier))

elif datatype == 'not_found':

return redirect('/not_found/{}'.format(identifier))

else:

db = get_db()

try:

chrom, pos, ref, alt = variant_str.split('-')

pos = int(pos)

# pos, ref, alt = get_minimal_representation(pos, ref, alt)

xpos = xbrowse.get_xpos(chrom, pos)

variant = lookups.get_variant(db, xpos, ref, alt)

if variant is None:

variant = {

'chrom': chrom,

'pos': pos,

'xpos': xpos,

'ref': ref,

'alt': alt

}

consequences = None

ordered_csqs = None

if 'vep_annotations' in variant:

variant['vep_annotations'] = order_vep_by_csq(variant['vep_annotations']) # Adds major_consequence

ordered_csqs = [x['major_consequence'] for x in variant['vep_annotations']]

ordered_csqs = reduce(lambda x, y: ','.join([x, y]) if y not in x else x, ordered_csqs, '').split(',') # Close but not quite there

consequences = defaultdict(lambda: defaultdict(list))

for annotation in variant['vep_annotations']:

annotation['HGVS'] = get_proper_hgvs(annotation)

consequences[annotation['major_consequence']][annotation['Gene']].append(annotation)

base_coverage = lookups.get_coverage_for_bases(db, xpos, xpos + len(ref) - 1)

any_covered = any([x['has_coverage'] for x in base_coverage])

metrics = lookups.get_metrics(db, variant)

print 'Rendering variant: %s' % variant_str

return render_template(

'variant.html',

variant=variant,

base_coverage=base_coverage,

consequences=consequences,

any_covered=any_covered,

ordered_csqs=ordered_csqs,

metrics=metrics

)

except Exception, e:

print 'Failed on variant:', variant_str, '; Error=', traceback.format_exc()

if gene_id in GENES_TO_CACHE:

return open(os.path.join(GENE_CACHE_DIR, '{}.html'.format(gene_id))).read()

else:

db = get_db()

try:

gene = lookups.get_gene(db, gene_id)

if gene is None:

abort(404)

cache_key = 't-gene-{}'.format(gene_id)

t = cache.get(cache_key)

if t is None:

variants_in_gene = lookups.get_variants_in_gene(db, gene_id)

transcripts_in_gene = lookups.get_transcripts_in_gene(db, gene_id)

# Get some canonical transcript and corresponding info

transcript_id = gene['canonical_transcript']

transcript = lookups.get_transcript(db, transcript_id)

variants_in_transcript = lookups.get_variants_in_transcript(db, transcript_id)

coverage_stats = lookups.get_coverage_for_transcript(db, transcript['xstart'] - EXON_PADDING, transcript['xstop'] + EXON_PADDING)

add_transcript_coordinate_to_variants(db, variants_in_transcript, transcript_id)

t = render_template(

'gene.html',

gene=gene,

transcript=transcript,

variants_in_gene=variants_in_gene,

variants_in_transcript=variants_in_transcript,

transcripts_in_gene=transcripts_in_gene,

coverage_stats=coverage_stats

)

cache.set(cache_key, t, timeout=1000*60)

print 'Rendering gene: %s' % gene_id

return t

except Exception, e:

print 'Failed on gene:', gene_id, ';Error=', e

db = get_db()

try:

transcript = lookups.get_transcript(db, transcript_id)

cache_key = 't-transcript-{}'.format(transcript_id)

t = cache.get(cache_key)

if t is None:

gene = lookups.get_gene(db, transcript['gene_id'])

gene['transcripts'] = lookups.get_transcripts_in_gene(db, transcript['gene_id'])

variants_in_transcript = lookups.get_variants_in_transcript(db, transcript_id)

coverage_stats = lookups.get_coverage_for_transcript(db, transcript['xstart'] - EXON_PADDING, transcript['xstop'] + EXON_PADDING)

add_transcript_coordinate_to_variants(db, variants_in_transcript, transcript_id)

t = render_template(

'transcript.html',

transcript=transcript,

transcript_json=json.dumps(transcript),

variants_in_transcript=variants_in_transcript,

variants_in_transcript_json=json.dumps(variants_in_transcript),

coverage_stats=coverage_stats,

coverage_stats_json=json.dumps(coverage_stats),

gene=gene,

gene_json=json.dumps(gene),

)

cache.set(cache_key, t, timeout=1000*60)

print 'Rendering transcript: %s' % transcript_id

return t

except Exception, e:

print 'Failed on transcript:', transcript_id, ';Error=', e

db = get_db()

try:

region = region_id.split('-')

cache_key = 't-region-{}'.format(region_id)

t = cache.get(cache_key)

if t is None:

chrom = region[0]

start = None

stop = None

if len(region) == 3:

chrom, start, stop = region

start = int(start)

stop = int(stop)

if start is None or stop - start > REGION_LIMIT:

return render_template(

'region.html',

genes_in_region=None,

variants_in_region=None,

chrom=chrom,

start=start,

stop=stop,

coverage=None

)

genes_in_region = lookups.get_genes_in_region(db, chrom, start, stop)

variants_in_region = lookups.get_variants_in_region(db, chrom, start, stop)

xstart = xbrowse.get_xpos(chrom, start)

xstop = xbrowse.get_xpos(chrom, stop)

coverage_array = lookups.get_coverage_for_bases(db, xstart, xstop)

t = render_template(

'region.html',

genes_in_region=genes_in_region,

variants_in_region=variants_in_region,

chrom=chrom,

start=start,

stop=stop,

coverage=coverage_array

)

print 'Rendering region: %s' % region_id

return t

except Exception, e:

print 'Failed on region:', region_id, ';Error=', e

db = get_db()

try:

variants = lookups.get_variants_by_rsid(db, rsid)

chrom = None

start = None

stop = None

print 'Rendering rsid: %s' % rsid

return render_template(

'region.html',

rsid=rsid,

variants_in_region=variants,

chrom=chrom,

start=start,

stop=stop,

coverage=None,

genes_in_region=None

)

except Exception, e:

print 'Failed on rsid:', rsid, ';Error=', e

return render_template(

'not_found.html',

query=query

if type(query) == str or type(query) == unicode:

unsupported = "TTN" if query.upper() in lookups.UNSUPPORTED_QUERIES else None

else:

unsupported = None

return render_template(

'error.html',

query=query,

unsupported=unsupported

db = get_db()

query = request.args.get('text')

datatype, identifier = lookups.get_awesomebar_result(db, query)

if datatype in ['gene', 'transcript']:

gene = lookups.get_gene(db, identifier)

link = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg19&position=chr%(chrom)s%%3A%(start)s-%(stop)s" % gene

output = '''Searched for %s. Found %s.

%s; Canonical: %s.

%s''' % (query, identifier, gene['full_gene_name'], gene['canonical_transcript'], link)

output += '' if 'omim_accession' not in gene else '''

In OMIM: %(omim_description)s

http://omim.org/entry/%(omim_accession)s''' % gene

return output

elif datatype == 'error' or datatype == 'not_found':

return "Gene/transcript %s not found" % query

else: