def demo3():
"""
You found a cool paper online and you want to find similar papers:
1. Download and parse the pdf
2. Compare to text of all publications in pubs_ database
3. Open the top 3 matches in browser (but note that current matching alg is
very basic and could be much improved)
Pre-requisites:
- Assumes 'pubs_nips' exists and contains pdf text inside
(under key 'pdf_text'). This can be obtained by running
nips_download_parse.py and then nips_add_pdftext.py
or by downloading it from site.
(https://sites.google.com/site/researchpooler/home)
Side-effects:
- will use os call to open a pdf with default program
"""
# fetch this pdf from website, parse it, and make a publication dict from it
# here is a random pdf from Andrew's website
url = 'http://ai.stanford.edu/~ang/papers/icml11-DeepEnergyModels.pdf'
print "downloading %s..." % (url, )
text = convertPDF(url) #extract the text
bow = stringToWordDictionary(
text) #extract the bag of words representation
p = {'pdf_text': bow} #create a dummy publication dict
# calculate similarities to our publications
print "loading database..."
pubs = loadPubs('pubs_nips')
print "computing similarities. (may take while with current implementation)"
scores = publicationSimilarityNaive(pubs, p)
# find highest scoring pubs
lst = [(s, i) for i, s in enumerate(scores) if s >= 0]
lst.sort(reverse=True)
# display top 50 matches
m = min(50, len(lst))
for s, i in lst[:m]:
print "%.2f is similarity to %s." % (s, pubs[i]['title'])
#open the top 3 in browser
print "opening the top 3..."
openPDFs([pubs[i]['pdf'] for s, i in lst[:3]])
def demo3():
"""
You found a cool paper online and you want to find similar papers:
1. Download and parse the pdf
2. Compare to text of all publications in pubs_ database
3. Open the top 3 matches in browser (but note that current matching alg is
very basic and could be much improved)
Pre-requisites:
- Assumes 'pubs_nips' exists and contains pdf text inside
(under key 'pdf_text'). This can be obtained by running
nips_download_parse.py and then nips_add_pdftext.py
or by downloading it from site.
(https://sites.google.com/site/researchpooler/home)
Side-effects:
- will use os call to open a pdf with default program
"""
# fetch this pdf from website, parse it, and make a publication dict from it
# here is a random pdf from Andrew's website
url = 'http://ai.stanford.edu/~ang/papers/icml11-DeepEnergyModels.pdf'
print "downloading %s..." % (url,)
text = convertPDF(url) #extract the text
bow = stringToWordDictionary(text) #extract the bag of words representation
p = {'pdf_text' : bow} #create a dummy publication dict
# calculate similarities to our publications
print "loading database..."
pubs = loadPubs('pubs_nips')
print "computing similarities. (may take while with current implementation)"
scores = publicationSimilarityNaive(pubs, p)
# find highest scoring pubs
lst = [(s, i) for i,s in enumerate(scores) if s>=0]
lst.sort(reverse = True)
# display top 50 matches
m = min(50, len(lst))
for s, i in lst[:m]:
print "%.2f is similarity to %s." % (s, pubs[i]['title'])
#open the top 3 in browser
print "opening the top 3..."
openPDFs([pubs[i]['pdf'] for s,i in lst[:3]])
try:
floc = p['pdf'].index('NIPS')
fname = p['pdf'][floc:]
txt = convertPDF('downloads/'+fname)
processed = True
print 'found %s in file!' % (p['title'],)
except:
pass
if not processed:
# download the PDF and convert to text
try:
print 'downloading pdf for [%s] and parsing...' % (p.get('title', 'an un-titled paper'))
txt = convertPDF(p['pdf'])
processed = True
print 'processed from url!'
except:
print 'error: unable to open download the pdf from %s' % (p['pdf'],)
print 'skipping...'
if processed:
# convert to bag of words and store
try:
p['pdf_text'] = stringToWordDictionary(txt)
except:
print 'was unable to convert text to bag of words. Skipped.'
print '%d/%d = %.2f%% done.' % (i+1, len(pubs), 100*(i+1.0)/len(pubs))
savePubs('pubs_nips', pubs_all)
