def _fetch_candles(params):
"""Fetch the given URL from OANDA and return a list of (utc-time, price).
Args:
params: A dict of URL params values.
Returns:
A sorted list of (time, price) points.
"""
url = '?'.join((URL, parse.urlencode(sorted(params.items()))))
logging.info("Fetching '%s'", url)
# Fetch the data.
response = net_utils.retrying_urlopen(url)
if response is None:
return None
data_string = response.read().decode('utf-8')
# Parse it.
data = json.loads(data_string, parse_float=D)
try:
# Find the candle with the latest time before the given time we're searching
# for.
time_prices = []
candles = sorted(data['candles'], key=lambda candle: candle['time'])
for candle in candles:
candle_dt_utc = datetime.datetime.strptime(
candle['time'],
r"%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=tz.tzutc())
candle_price = D(candle['openMid'])
time_prices.append((candle_dt_utc, candle_price))
except KeyError:
logging.error("Unexpected response data: %s", data)
return None
return sorted(time_prices)
def get_ft_symbol(self, security_type, exchange, ticker):
template = 'http://beta.morningstar.com/{}/{}/{}/quote.html'
url = template.format(security_type, exchange, ticker)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
except error.HTTPError:
return None
soup = BeautifulSoup(response, 'html.parser')
def make_finder(name):
def meta_finder(a):
return a.name == 'meta' and 'name' in a.attrs and a[
'name'] == name
return meta_finder
def get_meta(name):
attr = soup.find_all(make_finder(name))[0]
return attr['content']
fetched_exchange_id = get_meta('exchangeId')
fetched_ticker = get_meta('ticker')
sec_id = get_meta('secId')
return sec_id
def test_timeout_once(self):
response = http.client.HTTPResponse(mock.MagicMock())
response.status = 200
with mock.patch('urllib.request.urlopen', side_effect=[None,
response]):
self.assertIs(net_utils.retrying_urlopen('http://nowhere.com'),
response)
def get_historical_price(self, compound_ticker, date):
"""See contract in beancount.prices.source.Source."""
# security_type, exchange, ticker = compound_ticker.lower().split(':')
# symbol = self.get_ft_symbol(security_type, exchange, ticker)
symbol = compound_ticker
if not symbol:
logging.info("Could not find secId for %s" % compound_ticker)
return None
# Look back some number of days in the past in order to make sure we hop
# over national holidays.
begin_date = date - datetime.timedelta(days=5)
end_date = date
# template = 'http://mschart.morningstar.com/chartweb/defaultChart?type=getcc&secids={}&dataid={}&startdate={}&enddate={}¤cy=&format=1'
template = 'https://markets.ft.com/data/equities/ajax/get-historical-prices?startDate={}&endDate={}&symbol={}'
def fmt(d):
return d.strftime('%Y/%m/%d')
# symbol = 19753923
url = template.format(fmt(begin_date), fmt(end_date), symbol)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
response = json.loads(response)
if 'status' in response:
status = response['status']
if status['code'] != 200:
logging.info("HTTP Status: [%s] %s" %
(status['code'], status['message']))
return None
soup = BeautifulSoup(response['html'], 'html.parser')
except error.HTTPError:
return None
try:
entries = soup.find_all('td')
trade_date = entries[0].find_all('span')[0].contents[0]
trade_date = datetime.datetime.strptime(trade_date,
'%A, %B %d, %Y')
trade_date = trade_date.replace(tzinfo=pytz.UTC)
price = D(entries[4].contents[0])
return source.SourcePrice(price, trade_date, None)
except:
import sys
logging.error("Error parsing data.", sys.exc_info()[0])
return None
def get_csv(self, date):
template = 'https://firststatesuper.com.au/content/dam/ftc/superunitprices/super-{0:0>2}-{1}.csv'
url = template.format(date.month, date.year)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
else:
return response
except error.HTTPError:
return None
def get_historical_price(self, ticker, date):
"""See contract in beancount.prices.source.Source."""
# Look back some number of days in the past in order to make sure we hop
# over national holidays.
begin_date = date - datetime.timedelta(days=5)
end_date = date
# Make the query.
params = parse.urlencode(
sorted({
'q': ticker,
'startdate': begin_date.strftime('%b+%d,%Y'),
'enddate': end_date.strftime('%b+%d,%Y'),
'num': 5,
'output': 'csv',
}.items()))
url = 'http://www.google.com/finance/historical?{}'.format(params)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
data = response.read()
except socket.timeout:
logging.error("Connection timed out")
return None
except error.HTTPError:
# When the instrument is incorrect, you will get a 404.
return None
# Note: utf-8-sig automatically skips the BOM here.
data = data.decode('utf-8-sig').strip()
lines = data.splitlines()
assert len(lines) >= 2, "Too few lines in returned data: {}".format(
len(lines))
# Parse the header, find the column for the adjusted close.
columns = lines[0].split(',')
index_price = columns.index('Close')
assert index_price >= 0, "Could not find 'Adj Close' data column."
index_date = columns.index('Date')
assert index_date >= 0, "Could not find 'Date' data column."
# Get the latest data returned.
most_recent_data = lines[1].split(',')
close_price = D(most_recent_data[index_price])
date = datetime.datetime.strptime(most_recent_data[index_date],
'%d-%b-%y')
return source.SourcePrice(close_price, date, None)
def get_historical_price(self, ticker, date):
"""See contract in beancount.prices.source.Source."""
# Look back some number of days in the past in order to make sure we hop
# over national holidays.
begin_date = date - datetime.timedelta(days=5)
end_date = date
# Make the query.
params = parse.urlencode(
sorted({
's': ticker,
'a': begin_date.month - 1,
'b': begin_date.day,
'c': begin_date.year,
'd': end_date.month - 1,
'e': end_date.day,
'f': end_date.year,
'g': 'd',
'ignore': '.csv',
}.items()))
url = 'http://ichart.yahoo.com/table.csv?{}'.format(params)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
data = response.read().decode('utf-8').strip()
except error.HTTPError:
return None
if response is None:
return None
lines = data.splitlines()
assert len(lines) >= 2, "Too few lines in returned data: {}".format(
len(lines))
# Parse the header, find the column for the adjusted close.
columns = lines[0].split(',')
index_price = columns.index('Adj Close')
assert index_price >= 0, "Could not find 'Adj Close' data column."
index_date = columns.index('Date')
assert index_date >= 0, "Could not find 'Date' data column."
# Get the latest data returned.
most_recent_data = lines[1].split(',')
close_price = D(most_recent_data[index_price])
date = datetime.datetime.strptime(most_recent_data[index_date],
'%Y-%m-%d')
return source.SourcePrice(close_price, date, None)
def get_latest_price(self, ticker):
commodity, currency = ticker.split(':')
url = 'https://min-api.cryptocompare.com/data/price?fsym={}&tsyms={}'.format(
commodity, currency)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
response = json.loads(response)
except error.HTTPError:
return None
price = D(response[currency]).quantize(D('1.000000000000000000'))
trade_date = utils.default_tzinfo(datetime.now(), tz.gettz())
return source.SourcePrice(
D('0') if price == 0 else price, trade_date, currency)
def get_latest_price(self, ticker):
"""
Fetch the current price from Realt API
Args:
ticker: The ticker symbol of the cryptocurrency
Returns:
A SourcePrice object
"""
url = "https://api.realt.community/v1/token/{}".format(ticker)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode("utf-8").strip()
response = json.loads(response)
logging.info("Reponse: %s", response)
except error.HTTPError:
return None
logging.info("Price: %s", response["tokenPrice"])
logging.info("updatedate: %s", response["lastUpdate"]["date"])
trade_date = datetime.now()
trade_date = trade_date.replace(tzinfo=pytz.UTC)
logging.info("trade_date: %s", trade_date)
try:
price = D(response["tokenPrice"])
trade_date = datetime.now()
trade_date = trade_date.replace(tzinfo=pytz.UTC)
return (
None
if price == 0
else source.SourcePrice(
price, trade_date, response["currency"]
)
)
except Exception:
raise RealtError("Pas de cours disponible ?")
return None
def get_historical_price(self, ticker, date):
commodity, currency = ticker.split(':')
trade_date = datetime.combine(date, datetime.max.time())
trade_date = trade_date.replace(tzinfo=pytz.UTC)
ts = int(time.mktime(trade_date.timetuple()))
url = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym={}&tsyms={}&ts={}'.format(
commodity, currency, ts)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
response = json.loads(response)
except error.HTTPError:
return None
price = D(response[commodity][currency]).quantize(
D('1.000000000000000000'))
return source.SourcePrice(
D('0') if price == 0 else price, trade_date, currency)
def get_url(self, url_template, ticker):
app_id, currencies = ticker.split(':')
from_currency, to_currency = currencies.split('_')
url = url_template.format(app_id, from_currency, to_currency)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
response = json.loads(response)
except error.HTTPError:
return None
# we use quantize because otherwise the conversion from an float to a Decimal
# leaves tons of cruft (i.e. dozens of digits of meaningless precision) that
# just clutters up the price file
price = D(response['rates'][to_currency]).quantize(D('1.000000'))
trade_date = datetime.datetime.fromtimestamp(response['timestamp'], datetime.timezone.utc)
return source.SourcePrice(price, trade_date, from_currency)
def get_latest_price(self, ticker):
"""See contract in beancount.prices.source.Source."""
# Try "realtime" and just regular bid/ask pairs.
for fields, num_prices in [('l1d1', 1), ('b3b2d2', 2), ('b0a0d2', 2),
('p0d2', 1)]:
url = 'http://finance.yahoo.com/d/quotes.csv?s={}&f=c4{}'.format(
ticker, fields)
logging.info("Fetching %s", url)
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
data = response.read().decode('utf-8').strip()
except error.HTTPError:
return None
if data and not re.match('N/A', data):
break
else:
return None
components = data.split(',')
# Get the currency.
currency = components[0].strip('"')
# Get the
if num_prices == 1:
# Process just a price.
price = D(components[1])
else:
# Process separate bid/offer.
bid = D(components[1])
ask = D(components[2])
price = (bid + ask) / 2
# Get the trade date for that price.
trade_date = datetime.datetime.strptime(components[-1], '"%m/%d/%Y"')
return source.SourcePrice(price, trade_date, currency)
def test_max_retry(self):
with mock.patch('urllib.request.urlopen',
side_effect=[None, None, None, None, None, None]):
self.assertIsNone(net_utils.retrying_urlopen('http://nowhere.com'))
def test_success_other(self):
response = http.client.HTTPResponse(mock.MagicMock())
with mock.patch('urllib.request.urlopen', return_value=response):
self.assertIsNone(net_utils.retrying_urlopen('http://nowhere.com'))
def get_historical_price(self, compound_ticker, date):
"""See contract in beancount.prices.source.Source."""
security_type, exchange, ticker = compound_ticker.lower().split(':')
sec_id = self.get_morningstar_secid(security_type, exchange, ticker)
if not sec_id:
logging.info("Could not find secId for %s:%s:%s" %
(security_type, exchange, ticker))
return None
# Look back some number of days in the past in order to make sure we hop
# over national holidays.
begin_date = date - datetime.timedelta(days=5)
end_date = date
template = 'http://mschart.morningstar.com/chartweb/defaultChart?type=getcc&secids={}&dataid={}&startdate={}&enddate={}¤cy=&format=1'
def fmt(d):
return d.strftime('%Y-%m-%d')
# The data_id is a magic number that tells the Morningstar backend
# exactly what type of data you want. (i.e. price data, growth of $10,000,
# and so on) For some reason it uses a different number when talking about
# mutual funds vs etfs and stocks
if security_type == 'funds':
data_id = 8217
else:
data_id = 8225
url = template.format(sec_id, data_id, fmt(begin_date), fmt(end_date))
logging.info("Fetching %s", url)
current_tz = datetime.datetime.now(
datetime.timezone.utc).astimezone().tzinfo
def hook(dct):
""" An ad-hoc parser for Morningstar's weird format. """
if 'code' in dct:
return {'code': int(dct['code']), 'message': dct['message']}
elif set(dct.keys()) == set(('i', 'd')):
return {'i': int(dct['i']), 'd': dct['d']}
elif set(dct.keys()) == set(('i', 'v')):
return {
'i':
datetime.datetime.strptime(
dct['i'], '%Y-%m-%d').replace(tzinfo=current_tz),
'v':
D(dct['v'])
}
elif set(dct.keys()) == set(('i', 't')):
return dct
elif set(dct.keys()) == set('r'):
return dct
elif set(dct.keys()) == set(('data', 'status')):
return dct
else:
return dct
try:
response = net_utils.retrying_urlopen(url)
if response is None:
return None
response = response.read().decode('utf-8').strip()
response = json.loads(response, object_hook=hook)
data = response['data']
status = response['status']
if status['code'] != 200:
logging.info("HTTP Status: [%s] %s" %
(status['code'], status['message']))
return None
except error.HTTPError:
return None
try:
last_price = data['r'][-1]['t'][-1]['d'][-1]
price = last_price['v']
trade_date = last_price['i']
return source.SourcePrice(price, trade_date, None)
except:
import sys
logging.error("Error parsing data.", sys.exc_info()[0])
return None
def get_latest_price(self, ticker):
"""See contract in beancount.prices.source.Source."""
if ':' in ticker:
exchange, symbol = ticker.split(':')
else:
exchange = None
symbol = ticker
# Build the query.
params_dict = {
'q': symbol,
'f': 'd,c', # Date,Close
}
if exchange:
params_dict['x'] = exchange
# Always reach back 5 days in time because of long weekends.
if exchange in ('MUTF', 'MUTF_CA'):
params_dict['p'] = '5d'
else:
params_dict['p'] = '5d'
params_dict['i'] = 300 # secs, to get the most recent.
if exchange in ('TSE', 'MUTF_CA'):
quote_currency = 'CAD'
else:
quote_currency = 'USD'
url = 'http://www.google.com/finance/getprices?{}'.format(
parse.urlencode(sorted(params_dict.items())))
logging.info("Fetching %s", url)
# Fetch the data.
response = net_utils.retrying_urlopen(url)
if response is None:
return None
try:
data = response.read().decode('utf-8')
except socket.timeout:
logging.error("Connection timed out")
return None
data = parse.unquote(data).strip()
# Process the meta-data.
metadata = {}
lines = data.splitlines()
for index, line in enumerate(lines):
match = re.match('([A-Z_+]+)=(.*)$', line)
if not match:
break
metadata[match.group(1)] = match.group(2)
else:
# No data was found.
return None
# Initialize a custom timezone, if there was one.
try:
offset = int(metadata['TIMEZONE_OFFSET']) * 60
zone = tz.tzoffset("Custom", offset)
except KeyError:
zone = None
interval = int(metadata['INTERVAL'])
data_lines = lines[index:]
for line in data_lines:
# Process an update on timezone (I'm not sure if this will ever be
# seen, but we handle it).
match = re.match('TIMEZONE_OFFSET=(.*)', line)
if match:
# Associate an appropriately defined timezone matching that of
# the response. This is extra... we could just as well return a
# UTC time.
zone = tz.tzoffset("Custom", int(match.group(1)))
continue
time_str, price_str = line.split(',')
match = re.match('a(\d+)', time_str)
if match:
# Create time from the UNIX timestamp. Note: This must be
# initialized in UTC coordinates.
time_marker = datetime.datetime.fromtimestamp(
int(match.group(1)), tz.tzutc())
# Convert to the local timezone if required.
if zone is not None:
time_marker = time_marker.astimezone(zone)
time = time_marker
else:
# Add time as relative from previous timestamp.
seconds = int(time_str) * interval
time = time_marker + datetime.timedelta(seconds=seconds)
price = D(price_str)
return source.SourcePrice(price, time, quote_currency)
