def _process_authors(author_list):
"""Extract and process author data.
Parameters
----------
author_list : bs4.element.Tag
AuthorList tag, which contains tags related to author data.
Returns
-------
out : list of tuple of (str, str, str, str)
List of authors, each as (LastName, FirstName, Initials, Affiliation).
"""
# Pull out all author tags from the input
authors = extract(author_list, 'Author', 'all')
# Initialize list to return
out = []
# Extract data for each author
for author in authors:
out.append((extract(author, 'LastName',
'str'), extract(author, 'ForeName', 'str'),
extract(author, 'Initials',
'str'), extract(author, 'Affiliation', 'str')))
return out
def test_extract():
# Create a complex tag
out = bs4.element.Tag(name='Out')
inn1 = bs4.element.Tag(name='Inn')
inn2 = bs4.element.Tag(name='Inn')
inn1.append('words words')
inn2.append('more words')
out.append(inn1)
out.append(inn2)
# Test error - bad how
with raises(ValueError):
out_err = extract(out, 'Inn', 'bad')
# Test how = 'raw'
out_raw = extract(out, 'Inn', 'raw')
assert type(out_raw) is bs4.element.Tag
# Test how = 'str'
out_str = extract(out, 'Inn', 'str')
#TODO: Figure this out? Whats the return type?
#assert isinstance(out_str, str)
#assert out_str == 'words words'
# Test how = 'all'
out_all = extract(out, 'Inn', 'all')
assert type(out_all) is bs4.element.ResultSet
# Test with non-existent tag name
out_none = extract(out, 'bad', 'raw')
assert out_none is None
def _process_authors(author_list):
"""Extract and process author data.
Parameters
----------
author_list : bs4.element.Tag
AuthorList tag, which contains tags related to author data.
Returns
-------
out : list of tuple of (str, str, str, str)
List of authors, each as (LastName, FirstName, Initials, Affiliation).
"""
# Pull out all author tags from the input
authors = extract(author_list, 'Author', 'all')
# Initialize list to return
out = []
# Extract data for each author
for author in authors:
out.append((extract(author, 'LastName', 'str'), extract(author, 'ForeName', 'str'),
extract(author, 'Initials', 'str'), extract(author, 'Affiliation', 'str')))
return out
def _get_count(req, url):
"""Get the count of how many articles listed on search results URL.
Parameters
----------
url : str
URL to search with.
Returns
-------
count : int
Count of the number of articles found.
"""
# Request page from URL
page = req.get_url(url)
page_soup = BeautifulSoup(page.content, 'lxml')
# Get all count tags
counts = extract(page_soup, 'count', 'all')
try:
count = int(counts[0].text)
except IndexError:
count = 0
return count
def _scrape_papers(req, art_url, cur_dat):
"""Scrape information for each article found for a given term.
Parameters
----------
req : Requester() object
Manages request
art_url : str
URL for the article to be scraped
Returns
-------
cur_dat : Data() object
Object to store information for the current term.
"""
# Get page of all articles
art_page = req.get_url(art_url)
art_page_soup = BeautifulSoup(art_page.content, "xml")
# Pull out articles
articles = art_page_soup.findAll('PubmedArticle')
# Loop through each article, extracting relevant information
for ind, art in enumerate(articles):
# Get ID of current article
new_id = _process_ids(extract(art, 'ArticleId', 'all'), 'pubmed')
# Extract and add all relevant info from current articles to Data object
cur_dat = _extract_add_info(cur_dat, new_id, art)
return cur_dat
def _get_db_info(req, info_url):
"""Calls EInfo to get info and status of db to be used for scraping.
Parameters
----------
info_url : str
URL to request db information from.
Returns
-------
db_info : dict
Database information.
"""
# Get the info page and parse with BeautifulSoup
info_page = req.get_url(info_url)
info_page_soup = BeautifulSoup(info_page.content, 'lxml')
# Set list of fields to extract from eInfo
fields = ['dbname', 'menuname', 'description', 'dbbuild', 'count', 'lastupdate']
# Extract basic infomation into a dictionary
db_info = dict()
for field in fields:
db_info[field] = extract(info_page_soup, field, 'str')
return db_info
def _get_count(req, url):
"""Get the count of how many articles listed on search results URL.
Parameters
----------
url : str
URL to search with.
Returns
-------
count : int
Count of the number of articles found.
"""
# Request page from URL
page = req.get_url(url)
page_soup = BeautifulSoup(page.content, 'lxml')
# Get all count tags
counts = extract(page_soup, 'count', 'all')
try:
count = int(counts[0].text)
except IndexError:
count = 0
return count
def _scrape_papers(req, art_url, cur_dat):
"""Scrape information for each article found for a given term.
Parameters
----------
req : Requester() object
Manages request
art_url : str
URL for the article to be scraped
Returns
-------
cur_dat : Data() object
Object to store information for the current term.
"""
# Get page of all articles
art_page = req.get_url(art_url)
art_page_soup = BeautifulSoup(art_page.content, "xml")
# Pull out articles
articles = art_page_soup.findAll('PubmedArticle')
# Loop through each article, extracting relevant information
for ind, art in enumerate(articles):
# Get ID of current article
new_id = _process_ids(extract(art, 'ArticleId', 'all'), 'pubmed')
# Extract and add all relevant info from current articles to Data object
cur_dat = _extract_add_info(cur_dat, new_id, art)
return cur_dat
def _get_db_info(req, info_url):
"""Calls EInfo to get info and status of db to be used for scraping.
Parameters
----------
info_url : str
URL to request db information from.
Returns
-------
db_info : dict
Database information.
"""
# Get the info page and parse with BeautifulSoup
info_page = req.get_url(info_url)
info_page_soup = BeautifulSoup(info_page.content, 'lxml')
# Set list of fields to extract from eInfo
fields = [
'dbname', 'menuname', 'description', 'dbbuild', 'count', 'lastupdate'
]
# Extract basic infomation into a dictionary
db_info = dict()
for field in fields:
db_info[field] = extract(info_page_soup, field, 'str')
return db_info
def test_extract():
"""Test the extract function."""
# Create a complex tag
out = bs4.element.Tag(name='Out')
inn1 = bs4.element.Tag(name='Inn')
inn2 = bs4.element.Tag(name='Inn')
inn1.append('words words')
inn2.append('more words')
out.append(inn1)
out.append(inn2)
# Test error - bad how
with raises(ValueError):
out_err = extract(out, 'Inn', 'bad')
# Test how = 'raw'
out_raw = extract(out, 'Inn', 'raw')
assert type(out_raw) is bs4.element.Tag
# DROPPED CASE WITH MOVE TO PY35
# Test how = 'txt'
#out_txt = extract(out, 'Inn', 'txt')
#assert isinstance(out_txt, UnicodeType)
#assert out_txt == unicode('words words')
# Test how = 'str'
out_str = extract(out, 'Inn', 'str')
#TODO: Figure this out? Whats the return type?
#assert isinstance(out_str, str)
#assert out_str == 'words words'
# Test how = 'all'
out_all = extract(out, 'Inn', 'all')
assert type(out_all) is bs4.element.ResultSet
# Test with non-existent tag name
out_none = extract(out, 'bad', 'raw')
assert out_none is None
def _process_pub_date(pub_date):
"""Extract and process publication date data.
Parameters
----------
pub_date : bs4.element.Tag
PubDate tag, which contains tags with publication date information.
Returns
-------
year : int or None
Year the article was published.
month : str or None
Month the article was published.
"""
# Extract year, convert to int if not None
year = extract(pub_date, 'Year', 'str')
year = int(year) if year else year
# Extract month
month = extract(pub_date, 'Month', 'str')
return year, month
def _process_pub_date(pub_date):
"""Extract and process publication date data.
Parameters
----------
pub_date : bs4.element.Tag
PubDate tag, which contains tags with publication date information.
Returns
-------
year : int or None
Year the article was published.
month : str or None
Month the article was published.
"""
# Extract year, convert to int if not None
year = extract(pub_date, 'Year', 'str')
year = int(year) if year else year
# Extract month
month = extract(pub_date, 'Month', 'str')
return year, month
def _extract_add_info(cur_dat, new_id, art):
"""Extract information from article web page and add to
Parameters
----------
cur_dat : Data() object
Object to store information for the current term.
new_id : int
Paper ID of the new paper.
art : bs4.element.Tag() object
Extracted pubmed article.
Returns
-------
cur_dat : Data() object
Object to store data from the current term.
NOTES
-----
Data extraction is all in try/except statements in order to
deal with missing data, since fields may be missing.
"""
# Add ID of current article
cur_dat.add_id(new_id)
cur_dat.add_title(extract(art, 'ArticleTitle', 'str'))
cur_dat.add_authors(_process_authors(extract(art, 'AuthorList', 'raw')))
cur_dat.add_journal(extract(art, 'Title', 'str'),
extract(art, 'ISOAbbreviation', 'str'))
cur_dat.add_words(_process_words(extract(art, 'AbstractText', 'str')))
cur_dat.add_kws(_process_kws(extract(art, 'Keyword', 'all')))
cur_dat.add_pub_date(_process_pub_date(extract(art, 'PubDate', 'raw')))
cur_dat.add_doi(_process_ids(extract(art, 'ArticleId', 'all'), 'doi'))
# Increment number of articles included in Data
cur_dat.increment_n_articles()
return cur_dat
def _extract_add_info(cur_dat, new_id, art):
"""Extract information from article web page and add to
Parameters
----------
cur_dat : Data() object
Object to store information for the current term.
new_id : int
Paper ID of the new paper.
art : bs4.element.Tag() object
Extracted pubmed article.
Returns
-------
cur_dat : Data() object
Object to store data from the current term.
NOTES
-----
Data extraction is all in try/except statements in order to
deal with missing data, since fields may be missing.
"""
# Add ID of current article
cur_dat.add_id(new_id)
cur_dat.add_title(extract(art, 'ArticleTitle', 'str'))
cur_dat.add_authors(_process_authors(extract(art, 'AuthorList', 'raw')))
cur_dat.add_journal(extract(art, 'Title', 'str'), extract(art, 'ISOAbbreviation', 'str'))
cur_dat.add_words(_process_words(extract(art, 'AbstractText', 'str')))
cur_dat.add_kws(_process_kws(extract(art, 'Keyword', 'all')))
cur_dat.add_pub_date(_process_pub_date(extract(art, 'PubDate', 'raw')))
cur_dat.add_doi(_process_ids(extract(art, 'ArticleId', 'all'), 'doi'))
# Increment number of articles included in Data
cur_dat.increment_n_articles()
return cur_dat