def dois2articles(configLocation, doisAndValues):
# takes the location of the config file for mendeley, and a
# dictionary with DOIs and their respective values
# and returns a dictionary with DOIs as keys and
# abstracts, titles and more in a dict as values
if os.path.isfile(configLocation):
with open(configLocation) as f:
config = yaml.load(f)
mendeley = Mendeley(config["clientId"], config["clientSecret"])
session = mendeley.start_client_credentials_flow().authenticate()
dois = doisAndValues.keys()
doisAndInfo = {}
for doi in dois:
try:
info = session.catalog.by_identifier(doi=doi, view="stats")
doisAndInfo[doi] = {
"title": info.title,
"abstract": info.abstract,
"link": info.link,
"keywords": info.keywords,
"year": info.year
}
except Exception as e:
pass
return doisAndInfo
def genCatalog(self):
config_file = KEY
config = {}
with open('config.yml') as f:
config = yaml.load(f)
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
catalog_df = pd.read_csv(INPUT)
catalog_id_list = catalog_df['id'].tolist()
c = 0
authors_list = []
for catalog_id in catalog_id_list:
c += 1
print(c)
raw_catalog = session.catalog.get(catalog_id, view='all')
if raw_catalog.authors:
for author in raw_catalog.authors:
first_name = author.first_name
last_name = author.last_name
name = (first_name, last_name)
if name not in authors_list:
authors_list.append(name)
else:
pass
else:
pass
authors_df = pd.DataFrame()
authors_df['author_name'] = authors_list
return authors_df
class MendeleyRobot:
def __init__(self):
self.mendeley = Mendeley(config.MENDELEY_ID, config.MENDELEY_SECRET)
self.mendeley_session = self.mendeley.start_client_credentials_flow().authenticate()
def pdf_annotate(self, data):
filehash = data.get('filehash')
try:
log.info('looking up PDF in Mendeley')
doc = self.mendeley_session.catalog.by_identifier(filehash=filehash)
log.info('title...')
data.mendeley['title'] = doc.title
log.info('year...')
data.mendeley['year'] = doc.year
log.info('abstract...')
data.mendeley['abstract'] = doc.abstract
log.info('authors...')
data.mendeley['authors'] = [{'forename': a.first_name, 'lastname': a.last_name, 'initials': ''.join(name[0] for name in a.first_name.split())} for a in doc.authors]
except:
log.info('Unable to find PDF in Mendeley')
return data
def search(request):
if 'search' in request.GET:
print(cache)
try:
if request.GET['search'] in cache:
print("ok")
article_list = Article.objects.all().filter(
id__in=cache.get(request.GET['search']))
return render(request, 'frontend/search.html',
{'article_list': article_list})
except django.core.cache.backends.base.InvalidCacheBackendError:
pass
mendeley = Mendeley(settings.MENDELEY_ID, settings.MENDELEY_SECRET)
auth = mendeley.start_client_credentials_flow().authenticate()
request.session['token'] = auth.token
ids = get_data_by_query(auth.token['access_token'],
request.GET['search'], 50)
cache.set(request.GET['search'], ids, timeout=2000)
article_list = Article.objects.all().filter(id__in=ids)
return render(request, 'frontend/search.html',
{'article_list': article_list})
return redirect(reverse('home'))
def trx_abstracts(m):
TRX = MaltegoTransform()
doi = m.getProperty("DOI")
if not doi:
TRX.addUIMessage(
'A DOI is needed to perform this search. Please run "Search on Crossref" and, if a DOI is found, try again here'
)
return TRX.returnOutput()
client_id = Tokens.MendeleyID
client_secret = Tokens.MendeleySecret
mendeley = Mendeley(client_id, client_secret=client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
try:
abstract = session.catalog.by_identifier(doi=doi).abstract
except:
TRX.addUIMessage("Cannot find document on Mendeley")
else:
new = TRX.addEntity("me.Article", title)
new.addProperty("abstract", "Abstract", 'loose',
abstract.encode('utf-8'))
TRX.addUIMessage(abstract)
logging(TRX.returnOutput(), m.Maltegoxml)
return TRX.returnOutput()
def mendeleyAuth():
client_id = "8949"
client_secret = "u38KsziNTLHOD8VB"
mendeley = Mendeley(client_id, client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
return session
def mendeleyAuth():
client_id = "8910"
client_secret = "OxP464bB5roU81GH"
mendeley = Mendeley(client_id, client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
return session
def start_mendeley_session(mndly_config):
"""
Creates a new Mendeley session
:param mndly_config: Contains information 'client_id', 'secret'
:return: session
"""
mendeley = Mendeley(mndly_config['client_id'], mndly_config['secret'])
auth = mendeley.start_client_credentials_flow()
return auth.authenticate()
def trx_searchauthor(m):
TRX = MaltegoTransform()
authore = m.getProperty("person.fullname")
client_id = Tokens.MendeleyID
client_secret = Tokens.MendeleySecret
mendeley = Mendeley(client_id, client_secret=client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
page = session.catalog.advanced_search(author=authore).list()
if len(page.items) < 12:
limit = len(page.items)
else:
limit = m.Slider
# print m.Slider
# print limit
for n in range(limit):
doc = page.items[n]
titlee = clean_obsession(doc.title)
new = TRX.addEntity("me.Article", titlee)
new.setWeight(100 - n)
try:
new.addProperty("abstract", "Abstract", 'strict',
clean_obsession(doc.abstract).encode('utf-8'))
new.addProperty("year", "Year", 'strict', str(doc.year))
new.addProperty("DOI", "DOI", 'strict', doc.identifiers['doi'])
authori = [
doc.authors[n].first_name.encode('utf-8') + ' ' +
doc.authors[n].last_name.encode('utf-8')
for n in range(len(doc.authors))
]
new.addProperty("author", "Author", 'strict',
clean_obsession('; '.join(authori)))
if doc.keywords:
doc_keywords = [
doc.keywords[n].encode('utf-8')
for n in range(len(doc.keywords))
]
new.addProperty("keywords", "Keywords", 'strict',
clean_obsession('; '.join(doc_keywords)))
except:
# pass
# print 'o dear'
TRX.addUIMessage('Article: ' + titlee +
'. Not all fields could be downloaded')
# print titlee
# print TRX.returnOutput()
# print 'santa madonna'
logging(TRX.returnOutput(), m.Maltegoxml)
return TRX.returnOutput()
def init_mendeley_api():
with open('config.yml') as f:
config = yaml.load(f)
# These values should match the ones supplied when registering your application.
mendeley_object = Mendeley(config['clientId'], config['clientSecret'])
auth = mendeley_object.start_client_credentials_flow()
mendeley = auth.authenticate()
return mendeley
def startSession():
'''
Initializes credential flow and returns session object for search
'''
credentials = cred.getCred()
client_id = credentials['client_id']
client_secret = credentials['client_secret']
mendeley = Mendeley(client_id, client_secret=client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
print('Session open command sent')
return session
def _mendeley_data(doi):
"""Returns Mendeley data or 'None', retrieved by doi via Mendeley."""
config_file_name = gv.resources_path.joinpath('mendeley_user_config.yml').as_posix()
with open(config_file_name) as f:
config = yaml.load(f)
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
doc = session.catalog.by_identifier(doi=doi, view='bib')
mendeley_doi = doc.identifiers['doi']
if doi == mendeley_doi:
return doc
else:
raise DoiNotFoundException()
def avgRCSList(queryList):
client_id = "8910"
client_secret = "OxP464bB5roU81GH"
avgRCS =0
curryear = datetime.now().year #get the current year
mendeley = Mendeley(client_id=client_id, client_secret=client_secret)
resultList = []
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
queryList_subset = all_subset(queryList)
for query in queryList_subset:
x = 0
iterCount = 0
paperCount = 0
avgRCS = 0
avgyear = 0
pages = session.catalog.advanced_search(source=query, view="stats")
for page in pages.iter(page_size=100):
if iterCount == 1000:
break
if curryear - page.year <= 0:
x += 0
else:
x += page.reader_count / (curryear - page.year)
paperCount += 1
iterCount += 1
avgyear += page.year #calculate average year of publications
avgyear = avgyear / iterCount
avgRCS = round((x / paperCount),2)
resultList.append(avgRCS)
print(str(query) + ": " + str(avgRCS) + " | published: " + str(paperCount) + " | avgyear: " + str(round(avgyear)))
results = {
'subset':queryList_subset,
'marks':resultList,
'zipped':zip(queryList_subset,resultList),
}
return results
def mendeley_data_by_doi(doi):
# returns Mendeley data or 'None'
with open('mendeley_user_config.yml') as f:
config = yaml.load(f)
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
try:
doc = session.catalog.by_identifier(doi=doi, view='bib')
except MendeleyException:
return None
mendeley_doi = doc.identifiers['doi']
if doi == mendeley_doi:
return doc
# either an error or doi could not be resolved
return None
def load_articles_author(request):
if request.user.is_authenticated:
if request.method == 'POST':
author = Authors.objects.get(pk=request.POST['pk'])
mendeley = Mendeley(settings.MENDELEY_ID, settings.MENDELEY_SECRET)
auth = mendeley.start_client_credentials_flow().authenticate()
print(auth.token)
ids = get_article_from_authors(author.name,
auth.token['access_token'])
articles = Article.objects.filter(pk__in=ids)
return JsonResponse({
'id':
list(articles.values_list("pk", flat=True)),
'title':
list(articles.values_list("title", flat=True))
})
def get_article_from_data(request):
if request.user.is_authenticated:
if request.method == 'POST':
query = request.POST['query'].split(';')
mendeley = Mendeley(settings.MENDELEY_ID, settings.MENDELEY_SECRET)
auth = mendeley.start_client_credentials_flow().authenticate()
for i in query:
if query == '': continue
get_data_by_query(auth.token['access_token'], query=i)
if request.user.keywords:
for i in request.user.keywords:
get_data_by_query(auth.token['access_token'], query=i)
if request.user.authors:
for i in request.user.authors:
get_article_from_authors(i, auth.token['access_token'])
if request.user.tags:
for i in request.user.tags:
get_data_by_query(auth.token['access_token'], query=i)
return JsonResponse({"success": "Success"})
from mendeley import Mendeley
import yaml
import os
# Get the DOI to look up
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("doi", help="Searches the Mendeley catalogue for this DOI")
args = parser.parse_args()
config_file = 'config.yml'
config = {}
if os.path.isfile(config_file):
with open('config.yml') as f:
config = yaml.load(f)
else:
config['clientId'] = os.environ.get('MENDELEY_CLIENT_ID')
config['clientSecret'] = os.environ.get('MENDELEY_CLIENT_SECRET')
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
doi = args.doi
doc = session.catalog.by_identifier(doi=doi, view='stats')
print(dir(doc))
print '"%s" has %s readers.' % (doc.title, doc.reader_count)
class SDKCrawler(AbstractCrawler):
def __init__(self, app_id: str, app_secret: str):
self._app_id = app_id
self._app_secret = app_secret
self._initialized = False
self._mendeley = Mendeley(app_id, app_secret)
self._session = None
""":type : MendeleySession """
log.info(
"Intialized SDKCrawler with app_id: {app_id} and app_secret: {app_secret}"
.format(app_id=app_id, app_secret=app_secret))
def prepare(self):
try:
self._session = self._mendeley.start_client_credentials_flow(
).authenticate()
self._initialized = True
except Exception as e:
log.critical(e)
def destroy(self):
self._initialized = False
def get_group_members(self, group_id: str) -> [Member]:
if not self._initialized:
log.critical(
"get_group_members has been fired but the SDKCrawler was not initialized"
)
return []
results = []
members = self._session.group_members(group_id).iter()
for member in members:
m = get_member_from_json(member.member_json)
if m.role != 'follower':
results.append(m)
return results
def get_profile_by_id(self, profile_id: str) -> Profile:
if not self._initialized:
log.critical(
"get_profile_by_id has been fired but the SDKCrawler was not initialized"
)
return []
profile = self._session.profiles.get(profile_id)
return get_profile_from_json(profile.json)
def get_documents_by_profile_id(self, profile_id: str) -> [Document]:
if not self._initialized:
log.critical(
"get_documents_by_profile_id has been fired but the SDKCrawler was not initialized"
)
return []
results = []
"""
Unfortunately the official Mendeley SDK has no support for document queries by non-logged-in profile-ids
Therefore i'll hack around that and reuse the session object to authenticate my own call.
Critical SDK class:
https://github.com/Mendeley/mendeley-python-sdk/blob/master/mendeley/resources/documents.py
"""
documents = ExtendedDocuments(self._session).iter(
view='all', profile_id=profile_id, authored='true')
for document in documents:
d = get_document_from_json(document.json)
results.append(d)
return results
def get_documents_by_group_id(self, group_id: str) -> [Document]:
if not self._initialized:
log.critical(
"get_documents_by_group_id has been fired but the SDKCrawler was not initialized"
)
return []
results = []
documents = self._session.group_documents(group_id).iter(view='all')
for document in documents:
d = get_document_from_json(document.json)
results.append(d)
return results
def __init__(self, client_id, secret):
self._logger = logging.getLogger(__name__)
mendeley_object = Mendeley(client_id, secret)
auth = mendeley_object.start_client_credentials_flow()
self._mendeley = auth.authenticate()
from mendeley import Mendeley
import yaml
import os
# Get the DOI to look up
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("doi", help="Searches the Mendeley catalogue for this DOI")
args = parser.parse_args()
config_file = 'config.yml'
config = {}
if os.path.isfile(config_file):
with open('config.yml') as f:
config = yaml.load(f)
else:
config['clientId'] = os.environ.get('MENDELEY_CLIENT_ID')
config['clientSecret'] = os.environ.get('MENDELEY_CLIENT_SECRET')
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
doi = args.doi
doc = session.catalog.by_identifier(doi=doi, view='stats')
print '"%s" has %s readers.' % (doc.title, doc.reader_count)
class SDKCrawler(AbstractCrawler):
def __init__(self, app_id: str, app_secret: str):
self._app_id = app_id
self._app_secret = app_secret
self._initialized = False
self._mendeley = Mendeley(app_id, app_secret)
self._session = None
""":type : MendeleySession """
log.info("Intialized SDKCrawler with app_id: {app_id} and app_secret: {app_secret}".format(
app_id=app_id,
app_secret=app_secret
))
def prepare(self):
try:
self._session = self._mendeley.start_client_credentials_flow().authenticate()
self._initialized = True
except Exception as e:
log.critical(e)
def destroy(self):
self._initialized = False
def get_group_members(self, group_id: str) -> [Member]:
if not self._initialized:
log.critical("get_group_members has been fired but the SDKCrawler was not initialized")
return []
results = []
members = self._session.group_members(group_id).iter()
for member in members:
m = get_member_from_json(member.member_json)
if m.role != 'follower':
results.append(m)
return results
def get_profile_by_id(self, profile_id: str) -> Profile:
if not self._initialized:
log.critical("get_profile_by_id has been fired but the SDKCrawler was not initialized")
return []
profile = self._session.profiles.get(profile_id)
return get_profile_from_json(profile.json)
def get_documents_by_profile_id(self, profile_id: str) -> [Document]:
if not self._initialized:
log.critical("get_documents_by_profile_id has been fired but the SDKCrawler was not initialized")
return []
results = []
"""
Unfortunately the official Mendeley SDK has no support for document queries by non-logged-in profile-ids
Therefore i'll hack around that and reuse the session object to authenticate my own call.
Critical SDK class:
https://github.com/Mendeley/mendeley-python-sdk/blob/master/mendeley/resources/documents.py
"""
documents = ExtendedDocuments(self._session).iter(view='all', profile_id=profile_id, authored='true')
for document in documents:
d = get_document_from_json(document.json)
results.append(d)
return results
def get_documents_by_group_id(self, group_id: str) -> [Document]:
if not self._initialized:
log.critical("get_documents_by_group_id has been fired but the SDKCrawler was not initialized")
return []
results = []
documents = self._session.group_documents(group_id).iter(view='all')
for document in documents:
d = get_document_from_json(document.json)
results.append(d)
return results
def get_mendeley_session():
mendeley = Mendeley(config.MENDELEY_ID, config.MENDELEY_SECRET)
return mendeley.start_client_credentials_flow().authenticate()
def genCatalog(self):
config_file = KEY
config = {}
with open('config.yml') as f:
config = yaml.load(f)
mendeley = Mendeley(config['clientId'], config['clientSecret'])
session = mendeley.start_client_credentials_flow().authenticate()
id_list = []
# issn_list = []
# isbn_list = []
# pui_list = []
# sgr_list = []
scopus_list = []
doi_list = []
# pmid_list = []
title_list = []
type_list = []
year_list = []
authors_list = []
source_list = []
pages_list = []
volume_list = []
reader_count_list = []
link_list = []
dois = pd.read_csv(INPUT)['doi'].tolist()
for doi in dois:
print(len(doi_list))
# ['abstract', 'authors', 'chapter', 'city', 'content_type', 'day', 'edition', 'editors', 'file_attached', 'files', 'id', 'identifiers', 'institution', 'issue', 'keywords', 'link', 'month', 'pages', 'publisher', 'reader_count', 'reader_count_by_academic_status', 'reader_count_by_country', 'reader_count_by_subdiscipline', 'revision', 'series', 'source', 'title', 'type', 'volume', 'websites', 'year']
try:
raw_catalog = session.catalog.by_identifier(doi=doi,
view='all')
except:
pass
catalog_id = raw_catalog.id
# catalog_issn = raw_catalog.identifiers['issn']
# catalog_isbn = raw_catalog.identifiers['isbn']
# catalog_pui = raw_catalog.identifiers['pui']
# catalog_sgr = raw_catalog.identifiers['sgr']
if 'scopus' in raw_catalog.identifiers.keys():
catalog_scopus = raw_catalog.identifiers['scopus']
else:
catalog_scopus = 'NA'
catalog_doi = raw_catalog.identifiers['doi']
# catalog_pmid = raw_catalog.identifiers['pmid']
catalog_title = raw_catalog.title
catalog_type = raw_catalog.type
catalog_year = raw_catalog.year
if raw_catalog.authors:
author = raw_catalog.authors[0]
catalog_authors = (author.first_name, author.last_name)
else:
catalog_authors = 'NA'
catalog_source = raw_catalog.source
catalog_pages = raw_catalog.pages
catalog_volume = raw_catalog.volume
catalog_reader_count = raw_catalog.reader_count
catalog_link = raw_catalog.link
if catalog_title not in title_list:
id_list.append(catalog_id)
# issn_list.append(catalog_issn)
# isbn_list.append(catalog_isbn)
# pui_list.append(catalog_pui)
# sgr_list.append(catalog_sgr)
scopus_list.append(catalog_scopus)
doi_list.append(catalog_doi)
# pmid_list.append(catalog_pmid)
title_list.append(catalog_title)
type_list.append(catalog_type)
year_list.append(catalog_year)
authors_list.append(catalog_authors)
source_list.append(catalog_source)
pages_list.append(catalog_pages)
volume_list.append(catalog_volume)
reader_count_list.append(catalog_reader_count)
link_list.append(catalog_link)
else:
pass
catalog_df = pd.DataFrame()
catalog_df['id'] = id_list
# catalog_df['issn'] = issn_list
# catalog_df['isbn'] = isbn_list
# catalog_df['pui'] = pui_list
# catalog_df['sgr'] = sgr_list
catalog_df['scopus'] = scopus_list
catalog_df['doi'] = doi_list
# catalog_df['pmid'] = pmid_list
catalog_df['title'] = title_list
catalog_df['type'] = type_list
catalog_df['year'] = year_list
catalog_df['authors'] = authors_list
catalog_df['source'] = source_list
catalog_df['pages'] = pages_list
catalog_df['volume'] = volume_list
catalog_df['reader_count'] = reader_count_list
catalog_df['link'] = link_list
return catalog_df
def trx_keywords(m):
TRX = MaltegoTransform()
try:
keywords=[x for x in (m.getProperty("keywords").split('; '))]
keywords=keywords[:12]
except AttributeError:
TRX.addUIMessage('Silly, there are no keywords for this article. But you could insert them yourself and then try again!')
return TRX.returnOutput()
limit=12
# limit=m.getTransformSetting('HowMany')
print limit
# if not limit.isdigit():
# TRX.addUIMessage('Silly! That has to be a number')
# return TRX.returnOutput()
if limit>40:
TRX.addUIMessage("Sorry, that's too many! Currently the limit is 12 with the free Maltego, otherwise 50")
return TRX.returnOutput()
elif limit>12:
TRX.addUIMessage("You set a value > than 12. If you are using the free Maltego, you are still going to get 12")
return TRX.returnOutput()
client_id=Tokens.MendeleyID
client_secret= Tokens.MendeleySecret
mendeley = Mendeley(client_id, client_secret=client_secret)
auth = mendeley.start_client_credentials_flow()
session = auth.authenticate()
try:
pages = [session.catalog.search(keyword).list() for keyword in keywords]
except UnicodeEncodeError:
TRX.addUIMessage("""Sorry, Mendeley doesn't accept keywords with non-latin characters.
Please change them yourself and try again""")
return TRX.returnOutput()
n_results_per_key = limit / len(pages)
if n_results_per_key == 0:
n_results_per_key == 1
keyplace=0
for page in pages:
if len(page.items) < n_results_per_key:
limit=len(page.items)
else:
limit=n_results_per_key
for n in range(limit):
doc=page.items[n]
new = TRX.addEntity("me.Article", doc.title.encode('utf-8'))
new.setWeight(100-n)
# new.setLinkColor('0x808000')
new.setLinkLabel(keywords[keyplace].encode('utf-8'))
try:
new.addProperty("abstract","Abstract", True, doc.abstract.encode('utf-8'))
new.addProperty("year","Year", False, str(doc.year))
authori=[doc.authors[n].first_name.encode('utf-8')+' '+doc.authors[n].last_name.encode('utf-8') for n in range(0,len(doc.authors))]
new.addProperty("author","Author", True, '; '.join(authori))
doc_keywords=[doc.keywords[n].encode('utf-8') for n in range(len(doc.keywords))]
new.addProperty("keywords","Keywords",True, '; '.join(doc_keywords))
new.addProperty("DOI","DOI",'strict', doc.identifiers['doi'])
except:
TRX.addUIMessage('Article: '+doc.title.encode('utf-8')+'. Not all fields could be downloaded, probably keywords..')
keyplace+=1
logging(TRX.returnOutput(),m.Maltegoxml)
return TRX.returnOutput()
def index(request):
mendeley = Mendeley(8198, 'Fi7j614Pw3e42Hjo')
auth = mendeley.start_client_credentials_flow().authenticate()
request.session['token'] = auth.token
return redirect('listDocuments/')
