def papers_by_query_api(request):
if request.method == 'GET':
phrase = request.GET.get('phrase', '')
if not phrase:
return HttpResponseBadRequest()
query = SearchScholarQuery()
query.set_phrase(phrase)
querier = ScholarQuerier()
querier.send_query(query)
papers = querier.articles
if not papers:
result = {
'papers': [{
'title': '',
'id': 0,
'url': '',
'excerpt': ''
}]
}
else:
result = {
'papers': [{
'title': papers[0]['title'],
'id': papers[0]['cluster_id'],
'url': papers[0]['url'],
'excerpt': papers[0]['excerpt']
}]
}
return JsonResponse(result)
else:
return HttpResponseBadRequest()
def getRelatedPublications(author):
print author
settings = ScholarSettings() #adjust scholar settings
querier = ScholarQuerier() #Instance of ScholarQuerier() conducts a search on Google Scholar
querier.apply_settings(settings) #applies settings as provided by the instance of ScholarSettings()
query = SearchScholarQuery()
query.set_author(author)
querier.send_query(query)
print querier.articles
def getResult(query):
querier = ScholarQuerier()
citations = 0
url_citations = ""
clusterID = ""
try:
querier.send_query(query)
print querier.articles[0].attrs['cluster_id']
citations = querier.articles[0].attrs['num_citations'][0]
url_citations = querier.articles[0].attrs['url_citations'][0]
clusterID = querier.articles[0].attrs['cluster_id'][0]
except:
pass
return citations, url_citations, clusterID
def process(document):
scholar = ScholarQuerier()
query = SearchScholarQuery()
# save cookie at first paper
global save_cookie
if save_cookie:
query.set_phrase("quantum theory")
scholar.send_query(query)
scholar.save_cookies()
save_cookie = False
query.set_phrase(document.title)
scholar.send_query(query)
scholar_articles = scholar.articles
if len(scholar_articles) == 0:
return None
title_match_ratio = \
difflib.SequenceMatcher(None, document.title, scholar_articles[0]['title']).ratio()
if title_match_ratio < min_title_match_ratio:
return None
old_tags = document.tags
citation_tag = ncitations_to_tag(scholar_articles[0]['num_citations'])
new_tags = update_tags(old_tags, [(tag_pattern, citation_tag)])
new_tags.append(str(scholar_articles[0]['num_citations']))
document.update(tags=new_tags)
return scholar_articles[0]['num_citations']
def search(bot, update, args):
search_command = ' '.join(args)
bot.send_message(chat_id=update.message.chat_id, text="You searched for: " + search_command)
querier = ScholarQuerier()
query = SearchScholarQuery()
query.set_words(args)
querier.send_query(query)
articles = querier.articles
message = ""
bot.send_message(chat_id=update.message.chat_id, text="Number of results: " + str(len(articles)))
index = 0
for article in articles:
bot.send_message(chat_id=update.message.chat_id, text=str(index+1)+". " + article.attrs['title'][0])
def getPublications_Title(title): querier = ScholarQuerier() settings = ScholarSettings() querier.apply_settings(settings) query = SearchScholarQuery() publications = [] query.set_words(title) querier.send_query(query) related_list = scholar.json(querier) if related_list: print "No of related publications found : ", print len(related_list) for item in related_list: #print item.keys() #item["relatedTitle"] = title[0] publications.append(item) #time.sleep(random.randrange(10, 40, 2)); #time.sleep(60); return publications
def process(document):
scholar = ScholarQuerier()
query = SearchScholarQuery()
query.set_phrase(document.title)
scholar.send_query(query)
scholar_articles = scholar.articles
if len(scholar_articles) == 0:
return None
title_match_ratio = \
difflib.SequenceMatcher(None, document.title, scholar_articles[0]['title']).ratio()
if title_match_ratio < min_title_match_ratio:
return None
old_tags = document.tags
citation_tag = ncitations_to_tag(scholar_articles[0]['num_citations'])
new_tags = update_tags(old_tags, [(tag_pattern, citation_tag)])
document.update(tags=new_tags)
return scholar_articles[0]['num_citations']
def process(document):
scholar = ScholarQuerier()
query = SearchScholarQuery()
query.set_phrase(document.title)
scholar.send_query(query)
scholar_articles = scholar.articles
if len(scholar_articles) == 0:
return None
title_match_ratio = \
difflib.SequenceMatcher(None, document.title, scholar_articles[0]['title']).ratio()
if title_match_ratio < min_title_match_ratio:
return None
old_tags = document.tags
citation_tag = ncitations_to_tag(scholar_articles[0]['num_citations'])
new_tags = update_tags(old_tags, [(tag_pattern, citation_tag)])
document.update(tags=new_tags)
return scholar_articles[0]['num_citations']
def query_scholar_for_papers(author, searchstring):
querier = ScholarQuerier()
settings = ScholarSettings()
settings.set_citation_format(settings.CITFORM_BIBTEX)
settings.set_per_page_results(5)
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_author(author)
query.set_phrase(searchstring)
querier.send_query(query)
return_str = ''
if len(querier.articles) > 0:
return_str += querier.articles[0].as_citation() + '\n'
else:
return_str = 'Ooopsie. No results. Maybe we ran over the request limit?'
return return_str
def blocked():
print "Test if blocked...."
#time.sleep(random.randrange(10, 40, 2));
time.sleep(60);
publications = []
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_author("Ryan Baker")
querier.send_query(query)
related_list = scholar.json(querier)
if related_list:
print "Block Test : No of related publications found : ",
print len(related_list)
for item in related_list:
publications.append(item)
if len(publications) == 0:
return True
else:
return False
def find_article(canidates):
"""
A function that attempts to find a good match
from scholar.py for a set of canidate paper
titles, it returns the best result.
"""
canidate_scores = [5000 for dummy in canidates]
canidate_best_match = ['' for dummy in canidates]
querier = ScholarQuerier()
delay = 0
for ii, canidate in enumerate(canidates):
sleep(delay)
querier.UA = random.choice(USER_AGENTS)
querier.query(canidate)
for art in querier.articles:
title = art['title'].encode('ascii', 'ignore')
score = str_metric(canidate,
title)/max(len(title),len(canidate))
if score < canidate_scores[ii]:
canidate_scores[ii] = score
canidate_best_match[ii] = art
print '----------------------'
print 'Canidate: '+canidate
print 'Match: '+ \
canidate_best_match[ii]['title'].encode('ascii', 'ignore')
print 'Score '+str(canidate_scores[ii])
querier.clear_articles()
delay = max(random.gauss(30, 30), 5)
def cites_api(request):
if request.method == 'GET':
paper_id = request.GET.get('paper_id', 0)
page = request.GET.get('page', None)
if not paper_id or page is None:
return HttpResponseBadRequest()
query = CitesScholarQuery(paper_id, page)
querier = ScholarQuerier()
querier.send_query(query)
papers = querier.articles
cites = []
for paper in papers:
if not paper['cluster_id']:
continue
cites.append({'title': paper['title'],
'id': paper['cluster_id'],
'url': paper['url']})
return JsonResponse({'paper_id': paper_id, 'cites': cites})
else:
return HttpResponseBadRequest()
def papers_by_query_api(request):
if request.method == 'GET':
phrase = request.GET.get('phrase', '')
if not phrase:
return HttpResponseBadRequest()
query = SearchScholarQuery()
query.set_phrase(phrase)
querier = ScholarQuerier()
querier.send_query(query)
papers = querier.articles
if not papers:
result = {'papers': [{'title': '', 'id': 0, 'url': '', 'excerpt': ''}]}
else:
result = {'papers': [{'title': papers[0]['title'],
'id': papers[0]['cluster_id'],
'url': papers[0]['url'],
'excerpt': papers[0]['excerpt']}]}
return JsonResponse(result)
else:
return HttpResponseBadRequest()
def getPublications(authors): print authors querier = ScholarQuerier() settings = ScholarSettings() querier.apply_settings(settings) query = SearchScholarQuery() publications = [] for author in authors: if len(author) > 0: print "Using Author : ", print author query.set_author(author) querier.send_query(query) related_list = scholar.json(querier) if related_list: print "No of related publications found : ", print len(related_list) for item in related_list: #print item.keys() #item["relatedAuthor"] = author publications.append(item) #time.sleep(random.randrange(10, 40, 2)); time.sleep(20); return publications
def cites_api(request):
if request.method == 'GET':
paper_id = request.GET.get('paper_id', 0)
page = request.GET.get('page', None)
if not paper_id or page is None:
return HttpResponseBadRequest()
query = CitesScholarQuery(paper_id, page)
querier = ScholarQuerier()
querier.send_query(query)
papers = querier.articles
cites = []
for paper in papers:
if not paper['cluster_id']:
continue
cites.append({
'title': paper['title'],
'id': paper['cluster_id'],
'url': paper['url']
})
return JsonResponse({'paper_id': paper_id, 'cites': cites})
else:
return HttpResponseBadRequest()
def getPublications(author):
print author
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_author(author)
querier.send_query(query)
#scholar.csv(querier)
scholar.txt(querier, with_globals=False)
def getRelatedPublications(author):
print author
settings = ScholarSettings() #adjust scholar settings
querier = ScholarQuerier(
) #Instance of ScholarQuerier() conducts a search on Google Scholar
querier.apply_settings(
settings
) #applies settings as provided by the instance of ScholarSettings()
query = SearchScholarQuery()
query.set_author(author)
querier.send_query(query)
print querier.articles
def literature_search(query_terms, type='full_name'):
"""
perform a google scholar query with given terms
"""
querier = ScholarQuerier()
settings = ScholarSettings()
config = ScholarConf()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
querier.apply_settings(settings)
query = SearchScholarQuery()
papers = []
for item in query_terms.values:
repo_id = item[0]
if type !='full_name':
repo_name = item[1]
phrase = item[2]
keywords = item[3]
start_year = item[4]
if keywords:
if ',' not in keywords:
keywords = keywords + ','
query.set_words_some(keywords)
query.set_words(repo_name)
query.set_phrase(phrase)
phrase_text = repo_name + ', ' + phrase
else:
phrase = item[1]
start_year = item[2]
query.set_phrase(phrase) # commontk/CTK, meoyo/AIPS
phrase_text = phrase
print('search papers for {} ...'.format(phrase_text))
query.set_timeframe(start_year)
querier.send_query(query)
articles = querier.articles
if len(articles)==0:
continue
results = process_arts(config, item[0], phrase_text, articles)
papers = papers + results
time_delay = random.randrange(1,10)
time.sleep(time_delay)
return papers
def getPublications_Title(title):
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
publications = []
query.set_words(title)
querier.send_query(query)
related_list = scholar.json(querier)
if related_list:
print "No of related publications found : ",
print len(related_list)
for item in related_list:
#print item.keys()
#item["relatedTitle"] = title[0]
publications.append(item)
#time.sleep(random.randrange(10, 40, 2));
#time.sleep(60);
return publications
def query_scholar_for_papers(author, searchstring):
querier = ScholarQuerier()
settings = ScholarSettings()
settings.set_citation_format(settings.CITFORM_BIBTEX)
settings.set_per_page_results(5)
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_author(author)
query.set_phrase(searchstring)
querier.send_query(query)
return_str = ''
if len(querier.articles) > 0:
return_str += querier.articles[0].as_citation() + '\n'
else:
return_str = 'Ooopsie. No results. Maybe we ran over the request limit?'
return return_str
def blocked():
print "Test if blocked...."
#time.sleep(random.randrange(10, 40, 2));
time.sleep(60)
publications = []
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_author("Ryan Baker")
querier.send_query(query)
related_list = scholar.json(querier)
if related_list:
print "Block Test : No of related publications found : ",
print len(related_list)
for item in related_list:
publications.append(item)
if len(publications) == 0:
return True
else:
return False
def getPublications(authors):
print authors
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
publications = []
for author in authors:
if len(author) > 0:
print "Using Author : ",
print author
query.set_author(author)
querier.send_query(query)
related_list = scholar.json(querier)
if related_list:
print "No of related publications found : ",
print len(related_list)
for item in related_list:
#print item.keys()
#item["relatedAuthor"] = author
publications.append(item)
#time.sleep(random.randrange(10, 40, 2));
time.sleep(20)
return publications
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import yapbib.biblist as biblist
from scholar import ScholarQuerier
from . import db
q = ScholarQuerier()
COMMON_FIELDS = ('id', 'title', 'year', 'author', 'url', 'citation_count',
'version_count', 'citation_url', 'version_url', 'related_url')
class Article(object):
@staticmethod
def from_db(d):
if d is None:
return None
a = Article()
for name in COMMON_FIELDS:
assert hasattr(d, name), name
setattr(a, name, getattr(d, name))
return a
@property
def complete(self):
"""Whether article has complete information."""
for key in COMMON_FIELDS:
if not hasattr(self, key) or getattr(self, key) is None:
return False
import re
from pybtex.database import parse_bytes, parse_file
from scholar import ScholarQuerier, ScholarSettings, SearchScholarQuery
DST = Path(__file__).absolute().parents[1] / 'publications.bib'
IGNORE = """
vo2014cytotoxicity
takeilnatriureticpeptideisolatedfromeelbrain
matchintemporal
brodbeck2018transformation
""".split()
ACRONYMS = ['EEG', 'MEG', 'MRI']
querier = ScholarQuerier()
settings = ScholarSettings()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_phrase("eelbrain")
query.set_timeframe(2012, None)
query.set_include_patents(False)
bib = parse_file(DST, 'bibtex')
start = 0
while True:
querier.send_query(query)
if len(querier.articles) == 0:
break
return url
except:
return None
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='biburl.py')
parser.add_argument('-f',
'--bib_file',
type=str,
required=True,
help="Bibtex file <references.bib>")
args = parser.parse_args()
# Initialize scholar querier
querier = ScholarQuerier()
# Setup scholar settings
settings = ScholarSettings()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
# Apply settings
querier.apply_settings(settings)
# Load bib file
if not os.path.exists(args.bib_file):
raise IOException('File does not exist %s' % args.bib_file)
with open(args.bib_file) as bib_file:
bib_db = bibtexparser.load(bib_file)
for j, item in enumerate(bib_db.entries):
def get_results_for(title, author):
query = SearchScholarQuery()
query.set_author(author)
query.set_phrase(title)
query.set_num_page_results(1)
query.set_scope(True)
settings = ScholarSettings()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
querier = ScholarQuerier()
querier.apply_settings(settings)
querier.send_query(query)
for art in querier.articles:
print art.as_citation();
bibtex_split = art.as_citation().split("\n")
reftype = bibtex_split[0][1:-1].split("{")[0].lower();
refid = bibtex_split[0][1:-1].split("{")[1].lower();
bibtex_split.remove(bibtex_split[0])
#print reftype + " " + refid + " " + str(bibtex_split)
thismodule = sys.modules[__name__]
while(True):
try:
features_of_type = getattr(thismodule, reftype).func_code.co_varnames[ 1: getattr(thismodule, reftype).func_code.co_argcount ]
break;
except AttributeError:
var = raw_input("Type " + reftype + " not recongised, please enter a known type: ");
reftype = var;
while (True):
arranged_name = []
arranged_value = []
for i in range(1, 10):
arranged_name.append(None)
arranged_value.append(None)
for line in bibtex_split:
if ( line.find("=") > -1 ):
stored_name = line.split("=")[0].strip()
stored_value = line.split("=")[1].strip();
stored_value = stored_value[1:-(len(stored_value)-stored_value.rfind("}"))]
if stored_name in features_of_type:
arranged_name[features_of_type.index(stored_name)] = stored_name
arranged_value[features_of_type.index(stored_name)] = stored_value
short_arranged_name = arranged_name[ 0 : arranged_name.index(None)];
short_arranged_value = arranged_value[ 0 : arranged_value.index(None)];
if len(short_arranged_name) == len(features_of_type):
return getattr(thismodule, reftype)(refid, *short_arranged_value).__getprintable__(True)
else:
for feature in features_of_type:
if ( feature not in arranged_name ):
var = raw_input(feature + " is not provided by the retrieved bibtex entry. Would you like to enter it now? (Y) or (N)");
if var == "Y":
var = raw_input("Enter value for " + feature + ": ");
bibtex_split.append(feature + " = {" + var + "}");
ScholarConf.COOKIE_JAR_FILE = 'cookies.txt'
min_sleep_time_sec = 5
max_sleep_time_sec = 10
df = pd.read_csv("../data_seperate_sheet/Profile_Publications_Standard.csv")
#cites =[]
#citation_list = [] # [year of pub]
with open('citations_counts_25307.csv', "w") as csv_file:
writer = csv.writer(csv_file, delimiter=',')
for k,v in df['Article Title'][25307:].iteritems():
# why is df['Article Title'] of type dict??
querier = ScholarQuerier()
settings = ScholarSettings()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_phrase(v)
query.set_scope(True)
querier.send_query(query)
if querier.articles:
cites=querier.articles[0].__getitem__('num_citations')
citation_list=querier.articles[0].__getitem__('url_citations')
else:
cites=0
citation_list= ""
# json_results = []
from string import ascii_lowercase
import re
from pybtex.database import parse_bytes, parse_file
from scholar import ScholarQuerier, ScholarSettings, SearchScholarQuery
DST = Path(__file__).absolute().parents[1] / 'publications.bib'
IGNORE = """
vo2014cytotoxicity
takeilnatriureticpeptideisolatedfromeelbrain
matchintemporal
brodbeck2018transformation
""".split()
ACRONYMS = ['EEG', 'MEG', 'MRI']
querier = ScholarQuerier()
settings = ScholarSettings()
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_phrase("eelbrain")
query.set_timeframe(2012, None)
query.set_include_patents(False)
bib = parse_file(DST, 'bibtex')
start = 0
while True:
querier.send_query(query)
if len(querier.articles) == 0:
break
# extract articles
#input: full path to xlsx file containing article titles from scopus. Assumes that
#titles are first column in the sheet 'Qatar_Scopus'.
#output: csv file containing paper titles, citations, possibly free PDF link from web.
from openpyxl import load_workbook
import sys
from scholar import ScholarQuerier, ScholarSettings, SearchScholarQuery,onecsv
from time import sleep
import csv
querier = ScholarQuerier()
settings = ScholarSettings()
querier.apply_settings(settings)
query = SearchScholarQuery()
query.set_scope(True)
alldata=[]
counter=1
xlsxfile=sys.argv[1]
wb = load_workbook(xlsxfile, use_iterators=True)
print wb.get_sheet_names()
ws = wb.get_sheet_by_name('Qatar_Scopus')
for row in ws.iter_rows(row_offset=1):
if row[0].value is not None:
temp=[]
title=row[0].value.encode("utf-8")
query.set_phrase(title)
query.set_num_page_results(1)
querier.send_query(query)
x=onecsv(querier)
if (x!=None):
return False
# edit config.json first
mendeley = create_client()
if skip_documents:
print('Already tagged documents by mendeley_add_citations.py are skipped.')
else:
print('Processing all documents, including already tagged by mendeley_add_citations.py.')
print('See skip_documents variable in mendeley_add_citations.py to change this.\n')
print('Tags are added immediately. You can interrupt the script and continue later.\n')
print('citations\tyear\tMendeley library title')
num_skipped = 0
documents = mendeley.library(items=-1)
scholar = ScholarQuerier(count=1)
for docid in documents['document_ids']:
document = mendeley.document_details(docid)
if skip_documents and has_citation_tag(document['tags'], ['citations_.*']):
num_skipped = num_skipped+1
continue
try:
scholar.query(document['title'])
scholar_articles = scholar.articles
if len(scholar_articles) == 0:
print('No scholar articles found for ' + document['title'])
continue
except urllib2.HTTPError as e:
print e.msg