def get_folder(api, view): """Auxiliary function to get the cache folder belonging to a an API and eventually create the folder. """ if not config.has_section('Directories'): create_config() try: folder = config.get('Directories', api) except NoOptionError: folder = DEFAULT_PATHS[api] folder = os.path.join(folder, view or '') if not os.path.exists(folder): os.makedirs(folder) return folder
def get_folder(api, view): """Auxiliary function to get the cache folder belonging to an API, eventually create the folder. """ if not config.has_section('Directories'): create_config() try: folder = config.get('Directories', api) except NoOptionError: folder = DEFAULT_PATHS[api] config.set('Directories', api, folder) with open(CONFIG_FILE, 'w') as f: config.write(f) folder = os.path.join(folder, view or '') if not os.path.exists(folder): os.makedirs(folder) return folder
def get_content(url, params={}, *args, **kwds): """Helper function to download a file and return its content. Parameters ---------- url : string The URL to be parsed. params : dict (optional) Dictionary containing query parameters. For required keys and accepted values see e.g. https://api.elsevier.com/documentation/AuthorRetrievalAPI.wadl *args, **kwds : key-value parings, optional Keywords passed on to as query parameters. Must contain fields and values specified in the respective API specification. Raises ------ ScopusHtmlError or HTTPError If the status of the response is not ok. ValueError If the accept parameter is not one of the accepted values. Returns ------- resp : byte-like object The content of the file, which needs to be serialized. """ from simplejson import JSONDecodeError # Get credentials and set request headers key = config.get('Authentication', 'APIKey') header = { 'X-ELS-APIKey': key, 'Accept': 'application/json', 'User-Agent': user_agent } if config.has_option('Authentication', 'InstToken'): token = config.get('Authentication', 'InstToken') header.update({'X-ELS-APIKey': key, 'X-ELS-Insttoken': token}) # Perform request params.update(**kwds) # If config.ini has a section as follows: # # [Proxy] # https = protocol://server:port # # it uses a proxy as defined # see requests documentation for details if config.has_section("Proxy"): proxyDict = dict(config.items("Proxy")) resp = requests.get(url, headers=header, proxies=proxyDict, params=params) else: resp = requests.get(url, headers=header, params=params) # Try raising ScopusError with supplied error message # if no message given, do without supplied error message # at least raise requests error try: error_type = errors[resp.status_code] try: reason = resp.json()['service-error']['status']['statusText'] except (JSONDecodeError, KeyError): try: reason = resp.json()['message'] except: reason = "" raise errors[resp.status_code](reason) except KeyError: resp.raise_for_status() return resp
def get_content(url, params={}, *args, **kwds): """Helper function to download a file and return its content. Parameters ---------- url : string The URL to be parsed. params : dict (optional) Dictionary containing query parameters. For required keys and accepted values see e.g. https://api.elsevier.com/documentation/AuthorRetrievalAPI.wadl *args, **kwds : key-value parings, optional Keywords passed on to as query parameters. Must contain fields and values specified in the respective API specification. Raises ------ ScopusHtmlError or HTTPError If the status of the response is not ok. ValueError If the accept parameter is not one of the accepted values. Returns ------- resp : byte-like object The content of the file, which needs to be serialized. """ from random import shuffle from simplejson import JSONDecodeError # Set header, params and proxy keys = config.get('Authentication', 'APIKey').split(",") header = { 'X-ELS-APIKey': keys[0].strip(), 'Accept': 'application/json', 'User-Agent': user_agent } if config.has_option('Authentication', 'InstToken'): token = config.get('Authentication', 'InstToken') header['X-ELS-Insttoken'] = token params.update(**kwds) proxies = dict(config._sections.get("Proxy", {})) # Perform request, eventually replacing the current key resp = requests.get(url, headers=header, proxies=proxies, params=params) while resp.status_code == 429: try: keys.pop(0) # Remove current key shuffle(keys) header['X-ELS-APIKey'] = keys[0].strip() resp = requests.get(url, headers=header, proxies=proxies, params=params) config['Authentication']['APIKey'] = ",".join(list(keys)) except IndexError: # All keys depleted break # Eventually raise error, if possible with supplied error message try: error_type = errors[resp.status_code] try: reason = resp.json()['service-error']['status']['statusText'] except (JSONDecodeError, KeyError): try: reason = resp.json()['message'] except: reason = "" raise errors[resp.status_code](reason) except KeyError: resp.raise_for_status() return resp