def test_ScraperAPI(self):
proxy_generator = ProxyGenerator()
proxy_generator.ScraperAPI(os.getenv('SCRAPER_API_KEY'))
scholarly.set_timeout(60)
## Uses another method to test that proxy is working.
self.test_search_keyword()
def get_research_articles(self, max_num):
# Search string for Google Scholar to look for.
# e.g. "{self.title} {self.director.name}" would equate to "Concussion Peter Landesman" for the movie Concussion.
search_str = f'{self.title} {self.director.name}'
output = f""
try:
pg = ProxyGenerator()
ip = os.environ['PROXY_IP']
pg.SingleProxy(http=ip, https=ip)
o = scholarly.use_proxy(pg)
search_query = scholarly.search_pubs(search_str)
for i in range(0, max_num):
curr = next(search_query)
# For debugging purposes, this is how you pretty print the search query's contents.
#scholarly.pprint(curr)
# Grab the title of the article.
title = curr['bib']['title']
# Begin our formatted html output for each found research article.
output += f"""
<li>
"""
# See if a publication url (i.e. curr['pub_url']) exists. If so, add an external link to it.
if 'pub_url' in curr:
output += f"""
<a target='_blank' href=\"{curr['pub_url']}\">{title}</a>
"""
else:
output += f"""
{title}
"""
output += f"""
<br>
"""
# Writes the abstract (i.e.curr['bib']['abstract']) if it exists.
if 'bib' in curr and 'abstract' in curr['bib']:
output += f"""
<p>{curr['bib']['abstract']}</p>
"""
output += f"""
</li>
"""
except Exception as e:
pass
# Useful for seeing errors in your terminal. Replace pass with the print statement below.
#print(sys.stderr, e)
return output
def setUp(self):
proxy_generator = ProxyGenerator()
if "CONNECTION_METHOD" in scholarly.env:
self.connection_method = os.getenv("CONNECTION_METHOD")
else:
self.connection_method = "none"
if self.connection_method == "tor":
tor_sock_port = None
tor_control_port = None
tor_password = "******"
# Tor uses the 9050 port as the default socks port
# on windows 9150 for socks and 9151 for control
if sys.platform.startswith("linux") or sys.platform.startswith(
"darwin"):
tor_sock_port = 9050
tor_control_port = 9051
elif sys.platform.startswith("win"):
tor_sock_port = 9150
tor_control_port = 9151
proxy_generator.Tor_External(tor_sock_port, tor_control_port,
tor_password)
scholarly.use_proxy(proxy_generator)
elif self.connection_method == "tor_internal":
if sys.platform.startswith("linux"):
tor_cmd = 'tor'
elif sys.platform.startswith("win"):
tor_cmd = 'tor.exe'
proxy_generator.Tor_Internal(tor_cmd=tor_cmd)
scholarly.use_proxy(proxy_generator)
elif self.connection_method == "luminati":
scholarly.set_retries(10)
proxy_generator.Luminati(usr=os.getenv("USERNAME"),
passwd=os.getenv("PASSWORD"),
proxy_port=os.getenv("PORT"))
scholarly.use_proxy(proxy_generator)
elif self.connection_method == "freeproxy":
proxy_generator.FreeProxies()
scholarly.use_proxy(proxy_generator)
else:
scholarly.use_proxy(None)
def test_tor_launch_own_process(self):
"""
Test that we can launch a Tor process
"""
proxy_generator = ProxyGenerator()
if sys.platform.startswith("linux"):
tor_cmd = 'tor'
elif sys.platform.startswith("win"):
tor_cmd = 'tor.exe'
tor_sock_port = random.randrange(9000, 9500)
tor_control_port = random.randrange(9500, 9999)
result = proxy_generator.Tor_Internal(tor_cmd, tor_sock_port, tor_control_port)
self.assertTrue(result["proxy_works"])
self.assertTrue(result["refresh_works"])
self.assertEqual(result["tor_control_port"], tor_control_port)
self.assertEqual(result["tor_sock_port"], tor_sock_port)
# Check that we can issue a query as well
query = 'Ipeirotis'
scholarly.use_proxy(proxy_generator)
authors = [a for a in scholarly.search_author(query)]
self.assertGreaterEqual(len(authors), 1)
from scholarly import scholarly import yaml # something like ssh -D 9050 -q -C -N [email protected] from scholarly import scholarly, ProxyGenerator # default values are shown below proxies = { 'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050' } pg = ProxyGenerator() pg.SingleProxy(**proxies) scholarly.use_proxy(pg) # Retrieve the author's data, fill-in, and print #author=scholarly.search_author_id('4poYWhEAAAAJ') search_query = scholarly.search_author('Vassil Vassilev') while True: print("Iter") try: author = next(search_query).fill() if 'cern' in author.email: break #print(author) except StopIteration: break #sys.exit(1) print(author) print("Titles")
def set_proxy():
if ALLOW_PROXY_ON_SCHOLAR:
pg = ProxyGenerator()
pg.SingleProxy(http_proxy, https_proxy)
scholarly.use_proxy(pg)
from scholarly import scholarly, ProxyGenerator
import json
from dotenv import load_dotenv
from pathlib import Path
import os
env_path = Path('../') / '.env'
load_dotenv(dotenv_path=env_path)
SCRAPER = os.getenv("SCRAPER")
proxy_generator = ProxyGenerator()
proxy_generator.ScraperAPI(SCRAPER)
scholarly.set_timeout(60)
scholarly.use_proxy(proxy_generator)
search_query = scholarly.search_author('Maël Montévil')
author = scholarly.fill(next(search_query))
pubs = [
scholarly.fill(pub) for pub in author['publications']
if (pub['num_citations'] > 0)
]
pubs2 = [[pub, (list(scholarly.citedby(pub)))] for pub in pubs
if 'citedby_url' in pub]
print(json.dumps(pubs2, indent=2, default=lambda o: '<not serializable>'))
async def __call__(self):
UserCancel = KeyboardInterrupt
# region various embed types creation
def publication_embeds(result) -> discord.Embed:
embed = discord.Embed(
title=result["bib"]["title"],
description=result["bib"]["abstract"],
url=result["eprint_url"]
if "eprint_url" in result.keys()
else result["pub_url"],
)
embed.add_field(
name="Authors",
value=", ".join(result["bib"]["author"]).strip(),
inline=True,
)
embed.add_field(name="Publisher", value=result["bib"]["venue"], inline=True)
embed.add_field(
name="Publication Year", value=result["bib"]["pub_year"], inline=True
)
embed.add_field(
name="Cited By",
value=result["num_citations"]
if "num_citations" in result.keys()
else "0",
inline=True,
)
embed.add_field(
name="Related Articles",
value=f'https://scholar.google.com{result["url_related_articles"]}',
inline=True,
)
embed.set_footer(text=f"Requested by {self.ctx.author}")
return embed
def author_embeds(result) -> discord.Embed:
embed = discord.Embed(title=result["name"])
embed.add_field(
name="Cited By", value=f"{result['citedby']} articles", inline=True
)
embed.add_field(name="Scholar ID", value=result["scholar_id"], inline=True)
embed.add_field(
name="Affiliation",
value=result["affiliation"]
if "affiliation" in result.keys()
else "None",
inline=True,
)
embed.add_field(
name="Interests",
value=f"{', '.join(result['interests']) if 'interests' in result.keys() else 'None'}",
inline=True,
)
embed.set_image(url=result["url_picture"])
embed.set_footer(text=f"Requested by {self.ctx.author}")
return embed
def citation_embeds(result) -> discord.Embed:
embed = discord.Embed(
title=result["bib"]["title"],
description=f"```{scholarly.bibtex(result)}```",
url=result["eprint_url"]
if "eprint_url" in result.keys()
else result["pub_url"],
)
embed.set_footer(text=f"Requested by {self.ctx.author}")
return embed
# endregion
try:
# region user flags processing
pg = ProxyGenerator()
proxy = FreeProxy(rand=True, timeout=1, country_id=["BR"]).get()
pg.SingleProxy(http=proxy, https=proxy)
scholarly.use_proxy(pg)
# self.args processing
if self.args is None:
results = [next(scholarly.search_pubs(self.query)) for _ in range(5)]
embeds = list(map(publication_embeds, results))
elif "author" in self.args:
results = [
next(scholarly.search_author(self.query)) for _ in range(5)
]
embeds = list(map(author_embeds, results))
elif "cite" in self.args:
results = scholarly.search_pubs(self.query)
results = [results for _ in range(5)]
embeds = list(map(citation_embeds, results))
else:
await self.message.edit(content="Invalid flag")
return
# endregion
# sets the reactions for the search result
if len(embeds) > 1:
buttons = [[
{Button(style=ButtonStyle.grey, label="◀️", custom_id="◀️"): None},
{Button(style=ButtonStyle.red, label="🗑️", custom_id="🗑️"): None},
{Button(style=ButtonStyle.grey, label="▶️", custom_id="▶️"): None}
]]
else:
buttons = [[
Button(style=ButtonStyle.red, label="🗑️", custom_id="🗑️")
]]
await Sudo.multi_page_system(self.bot, self.ctx, self.message, tuple(embeds), buttons)
return
except asyncio.TimeoutError:
raise
except (asyncio.CancelledError, discord.errors.NotFound):
pass
except scholarly_exceptions._navigator.MaxTriesExceededException:
await self.message.edit(
content="Google Scholar is currently blocking our requests. Please try again later"
)
Log.append_to_log(self.ctx, f"{self.ctx.command} error", "MaxTriesExceededException")
return
except Exception as e:
await error_handler(self.bot, self.ctx, e, self.query)
finally:
return