def generate_theoretical_data(self,
ticker_tgt,
ticker_src,
step=0.00005,
pos_adj=None,
neg_adj=None):
"""Generates theoretical data for a stock based on another
stock.
Given two tickers, a granularity/precision step, and manual
offset/adjustments, generates more data for the first stock
(gen) to match the length of data in the second stock (src).
The generation is based on averages in existing real data and
assumes an existing correlation between two stocks (e.g. UPRO
and SPY supposedly have a correlation, or leverage factor of 3)
Args:
ticker_tgt: A ticker of the stock for which data should be
generated, i.e. the target for the generation
ticker_src: A ticker of the stock to be used as the data
source to aid in data generation.
NOTE: This implies the source data should be longer
than the data for the stock for which the generation
occurs
step: A value corresponding to a level of precision, or the
number of averages calculated and then used to generate
the data. NOTE: precision != accuracy and a default
value of 0.00005 is used if one is not given, based on
testing done on different values
pos_adj: A value to be used when adjusting movements in the
positive direction, i.e. a higher value will lead to
more pronounced positive moves (default: None, if None
a hardcoded default value will be used depending on
the ticker, typically 0)
neg_adj: A value to be used when adjusting movements in the
negative direction, i.e. a higher value will lead to
more pronounced negative moves (default: None, if None
a hardcoded default value will be used depending on
the ticker, typically 0)
Returns:
A tuple of price LUTs, one LUT containing real data
appended to a part of the generated data, the other
containing a full set of generated data. The former is
intended to be used in backtesting strategies, while the
latter is intended to be used for verifying generation
accuracy against existing real data.
"""
db = DataManager()
# get prices for tickers
price_lut_tgt = db.build_price_lut(ticker_tgt)
price_lut_src = db.build_price_lut(ticker_src)
# before doing any calculations, check if all data is on disk already
# NOTE: feature disabled for now, as it didnt respond to changes
# price_lut_gen_part = db.build_price_lut(ticker_tgt + '--GEN-PART')
# price_lut_gen_full = db.build_price_lut(ticker_tgt + '--GEN-FULL')
# if (len(price_lut_gen_part) == len(price_lut_src)
# and len(price_lut_gen_full) == len(price_lut_src)):
# return (price_lut_gen_part, price_lut_gen_full)
# sorted dates needed later
src_dates = sorted(price_lut_src.keys())
gen_dates = sorted(price_lut_tgt.keys())
# part of data will be real data
price_lut_gen_part = price_lut_tgt.copy()
# fully generated data needs a real point as an anchor
price_lut_gen_full = {gen_dates[0]: price_lut_tgt[gen_dates[0]]}
# a set of adjustments to use if not otherwise specified
adjustments = {
'UPRO': (0, 0),
'TMF': (0.01, 0.05),
'TQQQ': (0.025, 0),
'UDOW': (0, 0.01)
}
if step == 0.00005 and pos_adj is None and neg_adj is None:
try:
pos_adj = adjustments[ticker_tgt.upper()][0]
neg_adj = adjustments[ticker_tgt.upper()][1]
except KeyError:
pos_adj = 0
neg_adj = 0
# calculate % movements and leverage ratio, to use for the SA-LUT
moves = {}
ratios = {}
for i in range(len(gen_dates) - 1):
change_src = (
price_lut_src[gen_dates[i + 1]] / price_lut_src[gen_dates[i]] -
1)
change_gen = (
price_lut_tgt[gen_dates[i + 1]] / price_lut_tgt[gen_dates[i]] -
1)
moves[gen_dates[i + 1]] = change_src
if change_src == 0:
ratios[gen_dates[i + 1]] = 0.0
else:
ratios[gen_dates[i + 1]] = change_gen / change_src
sa_lut = SteppedAvgLookup(step, [moves[d] for d in gen_dates[1:]],
[ratios[d] for d in gen_dates[1:]])
# generate data going forward from gen data's anchor point
for i in range(len(gen_dates) - 1):
move = moves[gen_dates[i + 1]]
if move >= 0:
adj = pos_adj
else:
adj = neg_adj
price_lut_gen_full[gen_dates[i + 1]] = \
(price_lut_gen_full[gen_dates[i]]
* (move * (sa_lut.get(move) + adj) + 1))
# generate data going backwards from gen data's anchor point
for i in range(len(src_dates) - len(gen_dates) - 1, -1, -1):
move = (
price_lut_src[src_dates[i + 1]] / price_lut_src[src_dates[i]] -
1)
if move >= 0:
adj = pos_adj
else:
adj = neg_adj
gen_price = (price_lut_gen_full[src_dates[i + 1]] /
(move * (sa_lut.get(move) + adj) + 1))
price_lut_gen_full[src_dates[i]] = gen_price
price_lut_gen_part[src_dates[i]] = gen_price
# save data to disk for faster retrieval next time
db.write_stock_data(
ticker_tgt + '--GEN-FULL',
[[date, '-', '-', '-',
str(price_lut_gen_full[date]), '-']
for date in src_dates], False)
db.write_stock_data(
ticker_tgt + '--GEN-PART',
[[date, '-', '-', '-',
str(price_lut_gen_part[date]), '-']
for date in src_dates], False)
return (price_lut_gen_part, price_lut_gen_full)
class Market(object):
"""A Market containing stocks and a date.
Can be queried for stock prices at the current date of this Market
Attributes:
stocks: A map of stock tickers to price LUTs
new_period: A map of flags for market periods
dates: An array of dates for the market
date: A tuple containing (curr date index in dates, curr date)
Todo:
"""
def __init__(self, tickers=None, dates=None):
"""Intialize a Market with a set of dates and stock tickers
with corresponding price LUTs.
Args:
tickers: An array of tickers for which to build price LUTs
dates: An array of dates
"""
self._db = DataManager()
self.new_period = {'m': False, 'q': False, 'y': False}
self.commissions = 10
self.stocks = {}
self.stocks_indicators = {}
if tickers != None:
self.add_stocks(tickers)
self.dates = []
self.date = (-1, None)
if dates != None:
self.dates = dates
self.date = (0, self.dates[0])
def add_stocks(self, tickers):
"""Creates price LUTs and adds them to the Market. Also sets up
the data structures for indicators.
Args:
tickers: An array of tickers for which to create LUTs
"""
for ticker in tickers:
self.stocks[ticker.upper()] \
= self._db.build_price_lut(ticker.upper())
# create empty dict to be populated later by indicators
self.stocks_indicators[ticker.upper()] = {}
def add_indicator(self, ticker, indicator, indicator_lut):
"""Adds the indicator data for a ticker to this Market.
Args:
ticker: A ticker for which to add indicator data
indicator: A string for the indicator being added
indicator_lut: A lookup table for the indicator data itself
"""
self.stocks_indicators[ticker.upper()][indicator.upper()] = \
indicator_lut
def inject_stock_data(self, ticker, dates, prices, price_lut=None):
"""Injects provided stock data into this market.
Generally used for generated data, but can be used in any case
to bypass the default price LUT creation method.
Args:
ticker: A ticker for which to inject data
dates: An array of dates corresponding to the prices
prices: An array of prices corresponding to the dates
price_lut: A price lookup table with dates mapping to
prices to be used instead of building one from dates
and prices
"""
ticker = ticker.upper()
self.stocks_indicators[ticker] = {}
if price_lut:
self.stocks[ticker] = price_lut
return
price_lut = {}
for i in range(0, len(dates)):
price_lut[dates[i]] = prices[i]
self.stocks[ticker] = price_lut
def current_date(self):
"""Returns the current date of this Market.
Returns:
A string representing the current date in this Market
"""
return date_str(self.date[1])
def query_stock(self, ticker, num_days=0):
"""Query a stock at the current date.
Args:
ticker: A ticker to query
num_days: A value representing the number of days of prices
going backwards from the current date to return. A
value of 0 means only a float for today's value will be
returned. A value >0 means an array of that many values
will be returned (default: 0)
Returns:
A float representing the price of the stock
"""
ticker = ticker.upper()
if num_days:
dates = self.dates[
max(0, self.date[0] - num_days + 1):self.date[0] + 1]
return [float(self.stocks[ticker][date]) for date in dates]
try:
return float(self.stocks[ticker][self.current_date()])
except KeyError:
print("NEEDS FIX: no data for " + ticker + " at " + self.date[1])
return None
def query_stock_indicator(self, ticker, indicator):
"""Query a stock indicator value or set of values at the
current date.
Args:
ticker: A ticker to query
indicator: An identifying string for the indicator value to
return
Returns:
A float or set of floats representing the indicator
value(s)
"""
ticker = ticker.upper()
indicator = indicator.upper()
try:
return float(
self.stocks_indicators[ticker][indicator][self.current_date()])
except KeyError:
print('NEEDS FIX: no {} value for {} at {}'.format(
indicator, ticker, self.current_date()))
return None
def set_date(self, date):
"""Sets this Market to a given date.
Args:
date: A date to which to set this Market
"""
if date < self.dates[0]:
self.date = (0, self.dates[0])
return 0
if date > self.dates[-1]:
self.date = (len(self.dates) - 1, self.dates[-1])
return 0
try:
self.date = (self.dates.index(date), date)
return 0
except ValueError:
print("NEEDS FIX: date does not exist")
return 1
def set_default_dates(self):
"""Sets a default range for this Market's dates.
Based on existing stocks in this Market, decides an appropriate
range in which all stocks have prices.
"""
date_range = (date_str(dt.fromordinal(1)),
date_str(dt.fromordinal(999999)))
for price_lut in self.stocks.values():
dates = sorted(price_lut.keys())
date_range = (max(date_range[0], dates[0]), min(
date_range[1], dates[-1]))
date_idxs = (dates.index(
date_range[0]), dates.index(date_range[1]))
self.dates = dates[date_idxs[0]:date_idxs[1] + 1]
self.date = (0, self.dates[0])
def advance_day(self):
"""Advances this Market's date by one day."""
self.date = (self.date[0] + 1, self.dates[self.date[0] + 1])
self._raise_period_flags()
def _raise_period_flags(self):
"""Internal function to handle setting flags at new periods."""
last_date = date_obj(self.dates[self.date[0] - 1])
curr_date = date_obj(self.date[1])
self.new_period = {'m': False, 'q': False, 'y': False}
if last_date.year < curr_date.year:
self.new_period = {'m': True, 'q': True, 'y': True}
elif last_date.month != curr_date.month:
self.new_period['m'] = True
if (curr_date.month - 1) % 3 == 0:
self.new_period['q'] = True
