def _parse(res, n, url, params, verbose, *args, **kwds):
"""Auxiliary function to download results and parse json."""
cursor = "cursor" in params
if not cursor:
start = params["start"]
_json = res.get('search-results', {}).get('entry', [])
if verbose:
chunk = 1
# Roundup + 1 for the final iteration
chunks = int(n / params['count']) + (n % params['count'] > 0) + 1
print(f'Downloading results for query "{params["query"]}":')
print_progress(chunk, chunks)
# Download the remaining information in chunks
while n > 0:
n -= params["count"]
if cursor:
pointer = res['search-results']['cursor'].get('@next')
params.update({'cursor': pointer})
else:
start += params["count"]
params.update({'start': start})
resp = get_content(url, params, *args, **kwds)
data = resp.json()
_json.extend(data.get('search-results', {}).get('entry', []))
if verbose:
chunk += 1
print_progress(chunk, chunks)
return _json, resp.headers
def get_coauthors(self) -> Optional[List[NamedTuple]]:
"""Retrieves basic information about co-authors as a list of
namedtuples in the form
(surname, given_name, id, areas, affiliation_id, name, city, country),
where areas is a list of subject area codes joined by "; ".
Note: Method retrieves information via individual queries which will
not be cached. The Scopus API returns 160 coauthors at most.
"""
SIZE = 25
# Get number of authors to search for
url = self.coauthor_link
if not url:
return None
res = get_content(url, api="AuthorSearch")
data = loads(res.text)['search-results']
N = int(data.get('opensearch:totalResults', 0))
# Store information in namedtuples
fields = 'surname given_name id areas affiliation_id name city country'
coauth = namedtuple('Coauthor', fields)
coauthors = []
# Iterate over search results in chunks of `SIZE` results
count = SIZE
start = 0
while start < N:
params = {'start': start, 'count': count, 'accept': 'json'}
res = get_content(url, api="AuthorSearch", params=params)
data = loads(res.text)['search-results'].get('entry', [])
# Extract information for each coauthor
for entry in data:
aff = entry.get('affiliation-current', {})
try:
areas = [a['$'] for a in entry.get('subject-area', [])]
except TypeError: # Only one subject area given
areas = [entry['subject-area']['$']]
new = coauth(
surname=entry['preferred-name']['surname'],
given_name=entry['preferred-name'].get('given-name'),
id=int(entry['dc:identifier'].split(':')[-1]),
areas='; '.join(areas),
name=aff.get('affiliation-name'),
affiliation_id=aff.get('affiliation-id'),
city=aff.get('affiliation-city'),
country=aff.get('affiliation-country'))
coauthors.append(new)
start += SIZE
return coauthors or None
def get_coauthors(self):
"""Retrieves basic information about co-authors as a list of
namedtuples in the form
(surname, given_name, id, areas, affiliation_id, name, city, country),
where areas is a list of subject area codes joined by "; ".
Note: These information will not be cached and are slow for large
coauthor groups.
"""
SIZE = 25
# Get number of authors to search for
url = self.coauthor_link
res = get_content(url=url)
data = loads(res.text)['search-results']
N = int(data.get('opensearch:totalResults', 0))
# Store information in namedtuples
fields = 'surname given_name id areas affiliation_id name city country'
coauth = namedtuple('Coauthor', fields)
coauthors = []
# Iterate over search results in chunks of 25 results
count = SIZE
start = 0
while start < N:
params = {'start': start, 'count': count}
res = get_content(url=url, params=params, accept='json')
data = loads(res.text)['search-results'].get('entry', [])
# Extract information for each coauthor
for entry in data:
aff = entry.get('affiliation-current', {})
try:
areas = [a['$'] for a in entry.get('subject-area', [])]
except TypeError: # Only one subject area given
areas = [entry['subject-area']['$']]
new = coauth(
surname=entry['preferred-name']['surname'],
given_name=entry['preferred-name'].get('given-name'),
id=entry['dc:identifier'].split(':')[-1],
areas='; '.join(areas),
name=aff.get('affiliation-name'),
affiliation_id=aff.get('affiliation-id'),
city=aff.get('affiliation-city'),
country=aff.get('affiliation-country'))
coauthors.append(new)
start += SIZE
return coauthors or None
def __init__(self,
fname,
refresh,
params,
url,
download=None,
max_entries=None,
verbose=False,
*args,
**kwds):
"""Class intended as base class for superclasses.
Parameters
----------
fname : str
The filename (including path) of the cache object.
refresh : bool or int
Whether to refresh the cached file if it exists or not. If int
is passed, cached file will be refreshed if the number of days
since last modification exceeds that value.
params : dict
Dictionary used as header during the API request.
url : str
The URL to be accessed.
download : bool (optional, default=None)
Whether to download the query or not. Has no effect for
retrieval requests.
max_entries : int (optional, default=None)
Raise error when the number of results is beyond this number.
Has no effect for retrieval requests.
verbose : bool (optional, default=False)
Whether to print a progress bar for multip-page requests.
*args, **kwds
Arguments and key-value pairings to be passed on
to `get_content()`.
Raises
------
ScopusQueryError
If `refresh` is neither boolean nor numeric.
ValueError
If `refresh` is neither boolean nor numeric.
"""
# Compare age of file to test whether we refresh
refresh, exists, mod_ts = _check_file_age(fname, refresh)
# Read or dowload eventually with caching
search_request = "query" in params
if exists and not refresh:
self._mdate = mod_ts
if search_request:
with open(fname, 'rb') as f:
self._json = [loads(line) for line in f.readlines()]
self._n = len(self._json)
else:
with open(fname, 'rb') as f:
self._json = loads(f.read().decode('utf-8'))
else:
resp = get_content(url, params, *args, **kwds)
header = resp.headers
if search_request:
# Download results page-wise
res = resp.json()
n = int(res['search-results'].get('opensearch:totalResults',
0))
self._n = n
self._json = []
data = "".encode('utf-8')
cursor_false = "cursor" in params and not params["cursor"]
if cursor_false and n > max_entries:
# Stop if there are too many results
text = (f'Found {n} matches. Set max_entries to a higher '
f'number, change your query ({query}) or set '
'subscription=True')
raise ScopusQueryError(text)
if n and download:
data, header = _parse(res, n, url, params, verbose, *args,
**kwds)
self._json = data
else:
data = resp.text.encode('utf-8')
self._json = loads(data)
# Set private variables
self._mdate = time()
self._header = header
# Finally write data
_write_json(fname, data)
def __init__(self,
identifier,
api,
refresh,
view,
id_type=None,
date=None):
"""Class intended as superclass to perform retrievals.
Parameters
----------
identifier : str or int
A string of the query.
api : str
The name of the Scopus API to be accessed. Allowed values:
AbstractRetrieval, AuthorRetrieval, CitationOverview,
ContentAffiliationRetrieval.
refresh : bool
Whether to refresh the cached file if it exists or not.
view : str
The view of the file that should be downloaded.
id_type : str (optional, default=None)
The type of used ID.
Note: Will only take effect for the AbstractRetrieval API.
date : str (optional, default=None)
A string combining two years with a hyphen for which citations
should be looked up for.
Note: Will only take effect for the CitationOverview API.
Raises
------
ValueError
If the api parameter or view parameter is an invalid entry.
"""
# Checks
if api not in RETRIEVAL_URL:
raise ValueError('api parameter must be one of ' +
', '.join(RETRIEVAL_URL.keys()))
# Construct parameters
url = RETRIEVAL_URL[api]
if api == "AbstractRetrieval":
url += id_type + "/"
params = {'view': view}
if api == 'CitationOverview':
params.update({
'date': date,
'scopus_id': identifier.split('0-')[-1]
})
url += identifier
# Parse file contents
qfile = join(get_folder(api, view), identifier.replace('/', '_'))
res = get_content(qfile, refresh, url=url, params=params)
self._json = loads(res.decode('utf-8'))
self._view = view
def __init__(self,
params: Dict,
url: str,
api: str,
download: bool = True,
verbose: bool = False,
*args: str,
**kwds: str) -> None:
"""Class intended as base class for superclasses.
:param params: Dictionary used as header during the API request.
:param url: The URL to be accessed.
:param api: The Scopus API to be accessed.
:param download: Whether to download the query or not. Has no effect
for retrieval requests.
:param verbose: Whether to print a download progress bar.
:param args: Keywords passed on `get_content()`
:param kwds: Keywords passed on `get_content()`
Raises
------
ValueError
If `self._refresh` is neither boolean nor numeric.
"""
# Checks
try:
_ = int(self._refresh)
except ValueError:
msg = "Parameter refresh needs to be numeric or boolean."
raise ValueError(msg)
# Compare age of file to test whether we refresh
self._refresh, mod_ts = _check_file_age(self)
# Read or download, possibly with caching
fname = self._cache_file_path
search_request = "query" in params
if fname.exists() and not self._refresh:
self._mdate = mod_ts
if search_request:
self._json = [
loads(line) for line in fname.read_text().split("\n")
if line
]
self._n = len(self._json)
else:
self._json = loads(fname.read_text())
else:
resp = get_content(url, api, params, *args, **kwds)
header = resp.headers
if search_request:
# Get number of results
res = resp.json()
n = int(res['search-results'].get('opensearch:totalResults',
0))
self._n = n
self._json = []
# Results size check
if params.get("cursor") is None and n > SEARCH_MAX_ENTRIES:
# Stop if there are too many results
text = f'Found {n} matches. The query fails to return '\
f'more than {SEARCH_MAX_ENTRIES} entries. Change '\
'your query such that it returns fewer entries.'
raise ScopusQueryError(text)
# Download results page-wise
if download:
data = ""
if n:
data, header = _parse(res, n, url, api, params,
verbose, *args, **kwds)
self._json = data
else:
data = None
else:
data = loads(resp.text)
self._json = data
data = [data]
# Set private variables
self._mdate = time()
self._header = header
# Finally write data unless download=False
if download:
text = [dumps(item, separators=(',', ':')) for item in data]
fname.write_text("\n".join(text))