# Initialize a requester object
req = Requester()
###################################################################################################
# Set the minimum wait time between requests
req.set_wait_time(0.5)
###################################################################################################
# Use the Requester object to request some web pages
for url in [
'https://www.google.com', 'https://www.yahoo.com',
'https://duckduckgo.com'
]:
page = req.request_url(url)
print('Collecting web page \t {} \t got status code \t {}'.format(
page.url, page.status_code))
###################################################################################################
# Check details of the requester object
req.check()
###################################################################################################
# Get information from the requester object as a dictionary
print(req.as_dict())
###################################################################################################
# URLs Object
def collect_words(terms,
inclusions=[],
exclusions=[],
db='pubmed',
retmax=None,
field='TIAB',
usehistory=False,
api_key=None,
save_and_clear=False,
logging=None,
directory=None,
verbose=False):
"""Collect text data and metadata from EUtils using specified search term(s).
Parameters
----------
terms : list of list of str
Search terms.
inclusions : list of list of str, optional
Inclusion words for search terms.
exclusions : list of list of str, optional
Exclusion words for search terms.
db : str, optional, default: 'pubmed'
Which database to access from EUtils.
retmax : int, optional
Maximum number of articles to return.
field : str, optional, default: 'TIAB'
Field to search for term within.
Defaults to 'TIAB', which is Title/Abstract.
usehistory : bool, optional, default: False
Whether to use EUtils history, storing results on their server.
api_key : str, optional
An API key for a NCBI account.
save_and_clear : bool, optional, default: False
Whether to save words data to disk per term as it goes, instead of holding in memory.
logging : {None, 'print', 'store', 'file'}
What kind of logging, if any, to do for requested URLs.
directory : str or SCDB, optional
Folder or database object specifying the save location.
verbose : bool, optional, default: False
Whether to print out updates.
Returns
-------
results : list of Articles
Results from collecting data for each term.
meta_data : MetaData
Meta data from the data collection.
Notes
-----
The collection does an exact word search for the term given. It then loops through all
the articles found for that term.
For each article, it pulls and saves out data (including title, abstract, authors, etc),
using the hierarchical tag structure that organizes the articles.
"""
# Get EUtils URLS object, with desired settings, and build required utility URLs
urls = EUtils(db=db,
usehistory='y' if usehistory else 'n',
retmax=retmax,
retmode='xml',
field=field,
api_key=api_key)
urls.build_url('info', settings=['db'])
urls.build_url('search',
settings=['db', 'usehistory', 'retmax', 'retmode', 'field'])
urls.build_url('fetch', settings=['db', 'retmode'])
# Initialize results, meta data & requester
results = []
meta_data = MetaData()
req = Requester(wait_time=get_wait_time(urls.authenticated),
logging=logging,
directory=directory)
# Get current information about database being used
meta_data.add_db_info(get_db_info(req, urls.get_url('info')))
# Check inclusions & exclusions
inclusions = inclusions if inclusions else [[]] * len(terms)
exclusions = exclusions if exclusions else [[]] * len(terms)
# Loop through all the terms
for search, incl, excl in zip(terms, inclusions, exclusions):
# Collect term information and make search term argument
term = Term(search[0], search, incl, excl)
term_arg = mk_term(term)
if verbose:
print('Collecting data for: ', term.label)
# Initialize object to store data for current term articles
arts = Articles(term)
# Request web page
url = urls.get_url('search', settings={'term': term_arg})
page = req.request_url(url)
page_soup = BeautifulSoup(page.content, 'lxml')
if usehistory:
# Get number of articles, and keys to use history
count = int(page_soup.find('count').text)
web_env = page_soup.find('webenv').text
query_key = page_soup.find('querykey').text
# Loop through, collecting article data, using history
ret_start_it = 0
while ret_start_it < count:
# Set the number of articles per iteration (the ret_max per call)
# This defaults to 100, but will set to less if fewer needed to reach retmax
ret_end_it = min(100, int(retmax) - ret_start_it)
# Get article page, collect data, update position
url_settings = {
'WebEnv': web_env,
'query_key': query_key,
'retstart': str(ret_start_it),
'retmax': str(ret_end_it)
}
art_url = urls.get_url('fetch', settings=url_settings)
arts = get_articles(req, art_url, arts)
ret_start_it += ret_end_it
if ret_start_it >= int(retmax):
break
# Without using history
else:
ids = page_soup.find_all('id')
art_url = urls.get_url('fetch', settings={'id': ids_to_str(ids)})
arts = get_articles(req, art_url, arts)
arts._check_results()
if save_and_clear:
arts.save_and_clear(directory=directory)
results.append(arts)
meta_data.add_requester(req)
return results, meta_data