class Client():
company_forecast = Endpoint(
session=session,
url=("https://www.inderes.fi/fi/"
"rest/views/inderes_numbers_only_year_data.json?"),
headers_hook=headers_hook,
defaults={
"args_0": ",".join(ARGS_0),
"args_1": ",".join(ARGS_1)
},
tf_url=utils.compose(
requote_uri,
lambda url: (url.replace("args_0", "args[0]").replace(
"args_1", "args[1]"))),
tf_get_response=tf_company_response)
raw_json = Endpoint(
session=session,
url="https://www.inderes.fi/fi/osakevertailu",
headers_hook=headers_hook,
tf_get_response=utils.compose(
lambda d:
(d.get("inderes_ranking").get("company_gathered_data")),
json.loads, lambda res: (res.text.split("Drupal.settings,")[1].
split(");")[0].strip())))
data_table = Endpoint(
session=session,
url="https://www.inderes.fi/fi/osakevertailu",
headers_hook=headers_hook,
tf_get_response=utils.compose(
lambda data: data.astype({
'diff1d': float,
'diff1dprc': float,
'bidprice': float,
'askprice': float,
'lastprice': float,
'dayhighprice': float,
'daylowprice': float,
'closeprice1d': float,
'turnover': float,
'quantity': float,
'timestamp': float,
'this_month_millistream': float,
}), lambda x: pd.DataFrame(
index=[isin.get(v["isin"]) for v in x], data=x),
lambda d: list(d.values()), lambda d:
(d.get("inderes_ranking").get("company_gathered_data")),
json.loads, lambda res: (res.text.split("Drupal.settings,")[1].
split(");")[0].strip())))
def tf_company_response(res):
get_meta = utils.compose(
lambda x: pd.DataFrame(
index=[isin.get(v["isin"]) for v in x],
data=[{
"target_price": utils.safe_float(v.get("target_price")),
"suositus": v.get("suositus"),
"isin": v.get("isin")
} for v in x]), utils.listfilter(lambda x: x["year"] == "0"))
def create_get_year(year):
return utils.compose(
lambda x: pd.DataFrame(
index=[isin.get(v["isin"]) for v in x],
data=[
utils.dissoc(v, "target_price", "suositus", "isin")
for v in x
]).astype(float),
utils.listfilter(lambda x: x["year"] == year))
get_this = create_get_year(ARGS_1[1])
get_next = create_get_year(ARGS_1[2])
return utils.pipe(
res.json(), lambda x: {
"meta": get_meta(x),
"this": get_this(x),
"next": get_next(x)
})
def create_get_year(year):
return utils.compose(
lambda x: pd.DataFrame(
index=[isin.get(v["isin"]) for v in x],
data=[
utils.dissoc(v, "target_price", "suositus", "isin")
for v in x
]).astype(float),
utils.listfilter(lambda x: x["year"] == year))
def API():
"""Client for Paavo service for demographic data
User interface similar to ``stores.statfin``.
"""
session = requests.Session()
def PaavoEndpoint(url, tf_get_response, tf_post_params, tf_post_response):
return Endpoint(url=("http://pxnet2.stat.fi/PXWeb/api/v1/fi/"
"Postinumeroalueittainen_avoin_tieto/" + url),
tf_get_response=tf_get_response,
tf_post_params=tf_post_params,
tf_post_response=tf_post_response,
session=session)
def tf_post_params(query_code="Postinumeroalue",
query_selection_filter="item",
query_selection_values=["00920"],
response_format="json",
**kwargs):
return utils.update_dict(
{
"query": [{
"code": query_code,
"selection": {
"filter": query_selection_filter,
"values": query_selection_values
}
}],
"response": {
"format": response_format
}
}, kwargs)
tf_post_response = utils.compose(
lambda x: x.apply(pd.to_numeric, errors="coerce"),
lambda x: x.set_index("Tiedot"), lambda x: pd.DataFrame(
columns=[y["code"] for y in x["columns"]],
data=[y["key"] + y["values"] for y in x["data"]]),
lambda res: json.loads(codecs.decode(res.content, "utf-8-sig")))
@attr.s(frozen=True)
class Client():
# NOTE: There are variables 1, 2, ..., 9
# For more information, see http://www.stat.fi/org/avoindata/
# paikkatietoaineistot/paavo_en.html
#
all_variables_2018 = PaavoEndpoint(
url="2018/paavo_9_koko_2018.px",
tf_get_response=transform_get_response,
tf_post_params=tf_post_params,
tf_post_response=tf_post_response)
return Client()
def tf_latlon(params):
latlon = utils.compose(
lambda x: ({
"latlon": ",".join(utils.tuplemap(str)(x))
} if all(x) else {}), lambda x: (x.get("lat"), x.get("lon")))
return utils.pipe({
**params,
**latlon(params)
}, lambda x: utils.dissoc(x, "lat", "lon"))
class Client():
chart = Endpoint(
session=session,
url="https://query1.finance.yahoo.com/v8/finance/chart/",
tf_get_response=utils.compose(
lambda d: {
"meta": (extract(d).get("meta", {})),
"quotes":
pd.DataFrame(index=pd.to_datetime(
extract(d).get("timestamp", []), unit="s"),
data=utils.update_dict(
extract(d).get("indicators", {}).get(
"quote", [[]])[0],
extract(d).get("indicators", {}).get(
"adjclose", [[]])[0]))
}, lambda res: res.json()),
tf_get_resource=lambda ticker: ticker + "?",
defaults={
"range": "1mo",
"period1": -2208988800,
"period2": int(pd.Timestamp.now().timestamp()),
"interval": "1d",
"includePrePost": "false",
"events": "div,splits"
})
# TODO: Organize the response
financials = Endpoint(
session=session,
url="https://finance.yahoo.com/quote/",
tf_get_resource=lambda ticker: ticker + "/financials",
tf_get_response=utils.compose(
lambda text: (json.loads(text).get("context").get("dispatcher")
.get("stores").get("QuoteSummaryStore")),
lambda res: (res.text.split("root.App.main =")[1].split(
"(this)")[0].split(";\n}")[0].strip())))
holders = Endpoint(session=session,
url="https://finance.yahoo.com/quote/",
tf_get_resource=lambda ticker: ticker + "/holders",
tf_get_response=lambda res: pd.read_html(res.text))
def API():
"""Client for the Finnish Meteorological Institute weather API
Contains two endpoints:
* forecast_hirlam_surface_point_hourly_2d
* historical_forecast_hirlam_surface_point_hourly_2d
The service contains many more endpoints for Hirlam, Harmonie models,
weather station observations etc.. Also, data can be queried in various
formats. The present client endpoints support pointwise (latlon) queries
of local forecasts / histories from the precalculated Hirlam model
Parameters
----------
key : str
User key for accessing the API
Example
-------
.. code-block:: python
from stores.fmi import API
api = API()
data = api.forecast_hirlam_surface_point_hourly_2d.get(
lat=60.0, lon=20.0
)
Notes
-----
Currently unsure whether there is a lot of historical data available
through this service. Perhaps from other endpoints, but it seems
that the Hirlam multi-point coverage location-wise endpoint has only
up to past 6 hours of history available.
"""
# TODO: beautifulsoup4 and lxml to packages.yml
from bs4 import BeautifulSoup
session = requests.Session()
def WfsV2Endpoint(tf_response, url, tf_params):
return Endpoint(
url="https://opendata.fmi.fi/wfs?service=WFS&version=2.0.0&" + url,
defaults={"request": "getFeature"},
tf_get_response=tf_response,
tf_get_params=tf_params,
session=session)
def tf_latlon(params):
latlon = utils.compose(
lambda x: ({
"latlon": ",".join(utils.tuplemap(str)(x))
} if all(x) else {}), lambda x: (x.get("lat"), x.get("lon")))
return utils.pipe({
**params,
**latlon(params)
}, lambda x: utils.dissoc(x, "lat", "lon"))
strip_block = utils.compose(utils.listmap(lambda x: x.split(" ")),
utils.listmap(str.strip),
lambda x: x.string.strip().splitlines())
def parse_multipointcoverage_xml(xml_string):
soup = BeautifulSoup(xml_string, "xml")
index = utils.pipe(
soup.find("positions"),
strip_block,
utils.listmap(utils.listfilter(utils.safe_int)),
utils.listmap(lambda x: utils.safe_int(x[0]) if len(x) else None),
)
columns = [
tag.attrs.get("name")
for tag in soup.find("DataRecord").find_all("field")
]
data = utils.pipe(
soup.find("DataBlock").find("doubleOrNilReasonTupleList"),
strip_block, utils.listmap(utils.listmap(utils.safe_float)))
assert len(index) == len(data), "Index and values are incompatible"
return (index, data, columns)
tf_response = utils.compose(
lambda x: pd.DataFrame(
index=pd.to_datetime(x[0], unit="s"), data=x[1], columns=x[2]),
parse_multipointcoverage_xml, lambda x: x.text)
@attr.s(frozen=True)
class Client():
forecast_hirlam_surface_point_hourly_2d = WfsV2Endpoint(
tf_response=tf_response,
url=("storedquery_id="
"fmi::forecast::hirlam::surface::point::multipointcoverage&"
"timestep=60&"
"starttime={0}&"
"endtime={1}&").format(
(pd.Timestamp.now("UTC").floor("H").strftime(
'%Y-%m-%dT%H:%M:%SZ')),
((pd.Timestamp.now("UTC").floor("H") +
pd.Timedelta("47H")).strftime('%Y-%m-%dT%H:%M:%SZ'))),
tf_params=tf_latlon)
historical_forecast_hirlam_surface_point_hourly_2d = WfsV2Endpoint(
tf_response=tf_response,
url=("storedquery_id="
"fmi::forecast::hirlam::surface::point::multipointcoverage&"
"timestep=60&"
"starttime={0}&"
"endtime={1}&").format(
((pd.Timestamp.now("UTC").floor("H") -
pd.Timedelta("48H")).strftime('%Y-%m-%dT%H:%M:%SZ')),
(pd.Timestamp.now("UTC").floor("H").strftime(
'%Y-%m-%dT%H:%M:%SZ'))),
tf_params=tf_latlon)
return Client()
def API():
"""Client for StatFin data service
Example
-------
.. code-block:: python
from stores import statfin
api = statfin.API()
# Query data for a zip code
res = api.apartment_prices_quarterly.post(
query_code="Postinumero",
query_selection_values=["00940"],
)
# Query metadata
metadata = api.apartment_prices_quarterly.get()
metadata.Postinumero.valueTexts
# Query data for twenty zip codes using metadata
res = api.apartment_prices_quarterly.post(
query_code="Postinumero",
query_selection_values=metadata.Postinumero.values[:20]
)
NOTE: There seems to be an issue downloading all data for all zip codes at
once. The suggested method is to query data in chunks.
"""
session = requests.Session()
def ApartmentPricesEndpoint(url, tf_get_response, tf_post_params,
tf_post_response):
return Endpoint(url="http://pxnet2.stat.fi/PXWeb/api/v1/fi/StatFin/" +
url,
tf_get_response=tf_get_response,
tf_post_params=tf_post_params,
tf_post_response=tf_post_response,
session=session)
def tf_post_params(query_code="Postinumero",
query_selection_filter="item",
query_selection_values=["00920"],
response_format="json",
**kwargs):
return utils.update_dict(
{
"query": [{
"code": query_code,
"selection": {
"filter": query_selection_filter,
"values": query_selection_values
}
}],
"response": {
"format": response_format
}
}, kwargs)
tf_post_response = utils.compose(
lambda x: x.apply(lambda s: (pd.to_datetime(s) if s.name.startswith(
"Vuosi") else pd.to_numeric(s, errors="coerce"))), lambda x: pd.
DataFrame(columns=[y["code"] for y in x["columns"]],
data=[y["key"] + y["values"] for y in x["data"]]),
lambda res: json.loads(codecs.decode(res.content, "utf-8-sig")))
@attr.s(frozen=True)
class Client():
apartment_prices_quarterly = ApartmentPricesEndpoint(
url="asu/ashi/nj/statfin_ashi_pxt_112p.px?",
tf_get_response=transform_get_response,
tf_post_params=tf_post_params,
tf_post_response=tf_post_response)
apartment_prices_yearly = ApartmentPricesEndpoint(
url="asu/ashi/vv/statfin_ashi_pxt_112q.px?",
tf_get_response=transform_get_response,
tf_post_params=tf_post_params,
tf_post_response=tf_post_response)
return Client()