def testSearchDocuments(self):
"""Test search document logic
- try to add a study and fetch it using full-text search
"""
disease_name = "Pizza"
disease = Disease.get_or_insert(disease_name, name=disease_name)
obj = {"pubmed_id": "123", "name": "How is the pizza?", "disease_ref": disease, "disease_trait": "Pizza"}
study = Study.get_or_insert(
obj["pubmed_id"],
name=obj["name"],
disease_ref=obj["disease_ref"],
disease_trait=obj["disease_trait"],
pubmed_id=obj["pubmed_id"],
date=date.today(),
)
study.put()
AddStudyDocument(study)
result = Study.search_todict("123")
self.assertEqual(obj["name"], result[0]["name"]) # result[0], obj)
self.tSearchPage()
self.tSearchStudy()
self.tSearchDisease()
def get(self, pubmed_id):
self.setTemplate('studyview.html')
study = Study.get_by_key_name(pubmed_id)
if study is None:
self.redirect('/studies/')
return
self.out(study=study)
def get(self):
# check memcache for main
myfilter = self.request.get("filter", "") # returns name of disease to filter on
if myfilter != "" and myfilter is not None:
studies = Study.search_todict('"'+myfilter+'"')
return self.out(rendered = self.render("studylistrender.html", studies = studies, filter=myfilter))
page = self.request.get("page", "")
pagenr = None
if page == "":
pagenr = 1
else:
try:
pagenr = int(page)
except:
pagenr = 1
# get pagenumber from diseasequery
studies = self.studyquery.fetch_page(pagenr)
# rendered = memcache.get("studylist_0:50")
# if rendered is None:
# make large query, to check for speed when cached
# studies = Study.all().fetch(100)
rendered = self.render("studylistrender.html",studies=studies, myfilter=myfilter, page=pagenr, count=self.count)
# logging.debug(rendered )
# add to memchache
# if not memcache.add('studylist_0:50', rendered):
# logging.error("Memcache set failed.")
self.out(rendered=rendered)
def update_search():
"""Populate fulltext indexes"""
for gene in Gene.all().run():
AddGeneDocument(gene)
for study in Study.all().run():
AddStudyDocument(study)
for disease in Disease.all().run():
AddDiseaseDocument(disease)
for snp in Snp.all().run():
AddSNPDocument(snp)
print "Fulltext docs updated"
def post(self, pubmed_id):
self.setTemplate('studyview.html')
study = Study.get_by_key_name(pubmed_id)
if study is None:
self.redirect('/studies/')
return
comment = Comment()
comment.study = study.key()
comment.body = self.request.get("comment")
comment.user = UserData.current()
comment.date = datetime.now()
comment.put()
self.out(study=study)
def populate(path="gwascatalog.txt", limit=100):
"""Populate the database with data from gwascatalog.txt - one hell of an import!
- create models of SNP, Gene, Study and Disease and relations between the objects
- based on the following field-names in the TSV (excel):
Date Added to Catalog
PUBMEDID
First Author
Date
Journal
Link
Study
Disease_Trait
Initial Sample Size
Replication Sample Size
Region
Chr_id
Chr_pos
Reported Gene(s)
Mapped_gene
Upstream_gene_id
Downstream_gene_id
Snp_gene_ids
Upstream_gene_distance
Downstream_gene_distance
Strongest SNP-Risk Allele
SNPs
Merged
Snp_id_current
Context
Intergenic
Risk Allele Frequency
p-Value
Pvalue_mlog
p-Value (text)
OR or beta
95\% CI (text)
Platform [SNPs passing QC]
CNV"""
docs = csv.DictReader(open(path,'rb'), dialect='excel-tab')
pubids = {}
# read all GWAS into a dictionary, using pubmed_id as key
# - collecting all GWAS assorted with the same study under same key
for doc in docs:
pubid = doc["PUBMEDID"].strip()
if pubids.has_key(pubid):
pubids[pubid].append(doc)
else:
pubids[pubid] = [doc]
i = 0
for pid, lines in pubids.iteritems():
i += 1
if i == 200:
break
#print i
# create a new study object for each new iteration
# - use the first line to initiate the study model
init = lines[0]
# create or get study with study_id
study = Study.get_or_insert(pid,
name=init["Study"].strip(),
pubmed_id = pid) # disease_ref = disease,
# populate study with static data
# date = datetime.strptime(init["Date"].strip(), "%m/%d/%Y").date()
study.date = datetime.strptime(init["Date"].strip(), "%m/%d/%Y").date()
study.pubmed_url=init["Link"].strip()
study.init_sample = init["Initial Sample Size"].strip()
study.repl_sample= init["Replication Sample Size"].strip()
study.platform = init["Platform [SNPs passing QC]"].strip()
study.put()
disease_name = None
disease = None
for line in lines:
# if the disease in this GWAS row differs from the other
# ones in this study, create new disease relation:
tmp = line["Disease/Trait"].strip().lower()
if disease_name != tmp:
disease_name = tmp
disease = Disease.get_or_insert(disease_name,
name=disease_name)
study.add_disease(disease)
# A gwas has either a direct gene or a
# down-stream and up-stream gene
gene = None
up_gene = None
down_gene = None
# The retards at GWAS use 1 == intergenic, 2 == not intergenic
# ... no seriously, _retards_
intergenic = line["Intergenic"].strip() == "2"
if not intergenic:
geneid = line["Snp_gene_ids"].strip()
if geneid != "":
# not intergenic => one direct gene
names = line["Mapped_gene"].split(" - ")
if len(names) == 1:
gene = Gene.get_or_insert(geneid,
name = line["Mapped_gene"],
geneid = geneid)
if not study.key() in gene.studies:
gene.studies.append(study.key())
if not disease.key() in gene.diseases:
gene.diseases.append(disease.key())
gene.put()
else:
# up and downstream genes must be set
down_id = line["Downstream_gene_id"].strip()
up_id = line["Upstream_gene_id"].strip()
if up_id != "" and down_id != "":
up_down_names = line["Mapped_gene"].split(" - ")
if len(up_down_names) < 2:
# gene = NR / NS or whatever..
up_down_names = ["N/A", "N/A"]
# create upstream gene
up_name = up_down_names[0]
down_name = up_down_names[1]
# print down_name, up_name
# assert("-" not in up_name)
# assert("-" not in down_name)
assert(up_id != "")
assert(down_id != "")
up_gene = Gene.get_or_insert(up_id,
name = up_name,
geneid = up_id)
# up_gene.study.append(study.key())
down_gene = Gene.get_or_insert(down_id,
name = down_name,
geneid = down_id)
# up_gene.study.append(study.key())
# init snps..
snp = None
snpid = line["Snp_id_current"].strip()
if snpid != "":
# non=blank snp
try:
# create lineation only if a 'single' snp is in the gwas
int(snpid)
snp = Snp.get_or_insert(snpid,
snpid=snpid)
if gene:
snp.gene = gene
if not study.key() in snp.studies:
snp.studies.append(study.key())
if not disease.key() in snp.diseases:
snp.diseases.append(disease.key())
snp.put()
except:
# haplotype?
snpid = "N/A"
# if no gene or snp lineation is mentioned - ignore and just insert study
if (gene is None or up_gene is None) and snp is None:
print "skipping gwas"
continue
# init gwas
gwas = GWAS(study=study,
intergenic=intergenic,
disease=disease,
snp=snp)
# if SNP is intergenic, save up/down-stream, else gene
if intergenic:
if down_gene is not None and up_gene is not None:
gwas.downstream = down_gene
gwas.upstream = up_gene
else:
# could be None
gwas.gene = gene
# parse remaining gwas information
gwas.p_string = line["p-Value"].strip()
# could be none
gwas.snps = snpid
# parse out the exponent: 6E-8 => -8
try:
# test that p-Value is actually a float before parsing out
float(line["p-Value"])
gwas.p_val = int(line["p-Value"].split("E-")[1])
except Exception, e:
# forces the filter to downgrade this gwas wrt. p-value
gwas.p_val = 0
# could be interesting
gwas.strongest_snp_risk_allele = \
line["Strongest SNP-Risk Allele"].strip()
# gwas.CI = line["95% CI (text)"].strip()
gwas.OR_beta = line["OR or beta"].strip()
gwas.riscAlleleFrequency = \
line["Risk Allele Frequency"].strip()
gwas.put()
def __init__(self, *args, **kwargs):
super(studyList, self).__init__(*args, **kwargs)
self._template = 'baserender.html'
self.studyquery = PagedQuery(Study.all(), 10)
self.count = self.studyquery.page_count()
