def request_document(url,payload=None):
"""Using a document url, makes a get request and returnes the session data.
:param url: The url to send a get request to.
:type url: str
:returns: Returns the session.get() data as opppose to session.get().json() data.
"""
try:
res = Session.get(url,params=payload)
res.raise_for_status()
except requests.exceptions.HTTPError as message:
print(message)
return None
return res
def request_get(url,dataType='regular',payload=None):
"""For a given url and payload, makes a get request and returns the data.
:param url: The url to send a get request to.
:type url: str
:param dataType: Determines how far to drill down into the data. 'regular' returns the unfiltered data. \
'results' will return data['results']. 'pagination' will return data['results'] and append it with any \
data that is in data['next']. 'indexzero' will return data['results'][0].
:type dataType: Optional[str]
:param payload: Dictionary of parameters to pass to the url as url/?key1=value1&key2=value2.
:type payload: Optional[dict]
:returns: Returns the data from the get request.
"""
try:
res = Session.get(url,params=payload)
res.raise_for_status()
data = res.json()
except (requests.exceptions.HTTPError,AttributeError) as message:
print(message)
if (dataType == 'results' or dataType == 'pagination'):
return [None]
else:
return None
if (dataType == 'results'):
try:
data = data['results']
except KeyError as message:
print("{} is not a key in the dictionary".format(message))
return [None]
elif (dataType == 'pagination'):
counter = 2
nextData = data
try:
data = data['results']
except KeyError as message:
print("{} is not a key in the dictionary".format(message))
return [None]
if nextData['next']:
print('Found Additional pages.')
while nextData['next']:
try:
res = Session.get(nextData['next'])
res.raise_for_status()
nextData = res.json()
except:
print('Additional pages exist but could not be loaded.')
return(data)
print('Loading page '+str(counter)+' ...')
counter += 1
for item in nextData['results']:
data.append(item)
elif (dataType == 'indexzero'):
try:
data = data['results'][0]
except KeyError as message:
print("{} is not a key in the dictionary".format(message))
return None
except IndexError as message:
return None
return data