def pull_data(session: requests.sessions, token: str, url: str):
"""
Pulls data using the WaniKani RESTful API for an associated user and the collection/resource specified by the url.
:param session: Requests session object shared by different resources (see process_collections).
:param token: Authentication token associated with the user.
:param url: URL suffix associated with the desired collection or resource (e.g. reviews table, kanji info, etc.)
:return: Yields dictionary objects (parsed json) containing collection/resource data.
"""
headers = {
"Wanikani-Revision": "20170710"
} # May need to modify if api changes.
first_page = session.get(url=url, headers=headers,
auth=BearerAuth(token)).json()
yield first_page['data']
time.sleep(1)
# cursor based pagination
next_url = first_page['pages']['next_url']
while next_url is not None:
next_page = session.get(url=next_url,
headers=headers,
auth=BearerAuth(token)).json()
yield next_page['data']
next_url = next_page['pages']['next_url']
time.sleep(1)
def getSiteData(ses: requests.sessions, url2: str):
maxTries = 3
counter = 0
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0',
}
ses.headers = headers
# Try a few times if no data was given
while (page := ses.get(url2)).status_code != 200 and counter < maxTries:
time.sleep(1)
counter += 1
def get_event(event_id: Union[str, int],
webserver_address: str = GS_SERVER_ADDRESS,
geojson: bool = True,
session: requests.sessions = None) -> Dict:
"""
Takes an id and retrieves the corresponding precipitation event from
the web server. Simple implementation with requests for single event
retrieval.
Parameters
----------
event_id: Document containing events
webserver_address: Address of the web server
geojson: Whether to request in geojson format
session: Session object
Returns
-------
Event as json dictionary
"""
url = get_event_url(event_id, webserver_address, geojson)
response = session.get(url) if session else requests.get(url)
response.raise_for_status()
json_dict = json.loads(response.text)
return json_dict
def test_things_geojson_heatmap(
rsession: requests.sessions,
hostname: typing.AnyStr,
headers: typing.Dict,
):
params = {"min_lat": -1, "max_lat": 1, "min_lon": -1, "max_lon": 1}
url = f"{hostname}things_geojson_heatmap"
res = rsession.get(url, headers=headers, params=params)
response_dict = res.json()
assert response_dict["type"] == "FeatureCollection"
assert len(response_dict["features"]) > 0
async def getWeatherImage(s: requests.sessions, url: str):
imgUrl = "Default Image URL"
status = True
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0',
}
s.headers = headers
if idSearch := re.search(r"/c/([A-Za-z0-9]+)key.html", url):
imgUrl = imgUrlTemp.format(idSearch.group(1))
def test_things_leaflet_heatmap(
rsession: requests.sessions,
hostname: typing.AnyStr,
headers: typing.Dict,
):
params = {"min_lat": -1, "max_lat": 1, "min_lon": -1, "max_lon": 1}
url = f"{hostname}things_leaflet_heatmap"
res = rsession.get(url, headers=headers, params=params)
response_dict = res.json()
assert len(response_dict["data"]) > 0
assert response_dict.get("total") is not None
assert response_dict.get("max_value") is not None
assert response_dict.get("num_docs") is not None
def test_get_thing(
rsession: requests.sessions,
hostname: typing.AnyStr,
headers: typing.Dict,
authority_id: typing.AnyStr,
format: typing.AnyStr,
):
first_thing = _get_things_with_authority(rsession, hostname, headers,
authority_id, 1)[0]
first_thing_id = first_thing["id"]
url = f"{hostname}thing/{first_thing_id}"
params = {"format": format}
res = rsession.get(url, headers=headers, params=params)
response_dict = res.json()
assert len(response_dict) > 0
def _get_things_with_authority(
rsession: requests.sessions,
hostname: typing.AnyStr,
headers: typing.Dict,
authority_id: typing.AnyStr,
limit: int,
) -> typing.List:
url = f"{hostname}thing/"
params = {
"limit": limit,
"authority": authority_id,
"status": 200,
}
res = rsession.get(url, headers=headers, params=params)
response_dict = res.json()
things = response_dict.get("data")
return things