def test_most_recent_date(self):
# Most recent date should be the most recent candle in the db
num_symbols = len(get_symbols())
num_candles = 10 * num_symbols
max_date = get_max_from_column(column='open_date')
from_date = max_date - timedelta(hours=10)
engineered_data = core.engineer_data(from_date=from_date)
assert engineered_data.open_date.max() == max_date
def test_earliest_date(self):
# Earliest engineered date should be from_date
num_symbols = len(get_symbols())
num_candles = 10 * num_symbols
max_date = get_max_from_column(column='open_date')
from_date = max_date - timedelta(hours=10)
engineered_data = core.engineer_data(from_date=from_date)
assert engineered_data.open_date.min() == from_date
def test_num_candles(self):
# Number of candles should be proportional to number of hours from from_date
num_symbols = len(get_symbols())
num_candles = 11 * num_symbols
max_date = get_max_from_column(column='open_date')
from_date = max_date - timedelta(hours=10)
engineered_data = core.engineer_data(from_date=from_date)
assert len(engineered_data) == num_candles
def update_candles(debug=False):
"""
Insert candles from the most recent open_date in the database to
the current time.
"""
symbols = db.get_symbols()
startTime = db.get_max_from_column(column='open_date')
if debug:
return insert_hourly_candles(symbols, startTime=startTime, debug=True)
else:
insert_hourly_candles(symbols,
startTime=startTime,
debug=False,
verbose=True)
def insert_engineered_data(verbose=True):
# Get indicators from subclasses
indicators = list(CustomIndicator.__subclasses__())
# Get columns for comparison to incoming indicators
sql = 'SHOW COLUMNS IN engineered_data;'
candle_cols = list(Database().execute(sql).Field)
# TODO add_column assumes
for indicator in indicators:
if indicator.__name__ not in candle_cols:
db.add_column('engineered_data', indicator.__name__, 'float(20,9)')
# Get starting date for insert
from_date = db.get_max_from_column(table='engineered_data',
column='open_date')
# If there's nothing in the table, populate the entire thing
if not from_date:
from_date = db.get_min_from_column(column='open_date')
ins = engineer_data(from_date=from_date, verbose=verbose)
# Convert to sql-friendly dates
convert_to_sql = lambda x: DateConvert(x).date
ins.open_date = ins.open_date.map(convert_to_sql)
ins.close_date = ins.close_date.map(convert_to_sql)
if verbose:
print('Inserting engineered data into database...')
Database().insert('engineered_data',
ins,
verbose=verbose,
auto_format=True)
def engineer_data(from_date = None, verbose=False):
"""
Get candles from database, add custom indicators.
Parameters:
---------------
from_date: UTC datetime, datestring, or second timestamp.
The date at which to pull data from
verbose: boolean
True to print a progress bar.
"""
def interpolate_nulls(candles):
candles['interpolated'] = False
null_inds = pd.isnull(candles).any(1).nonzero()[0]
if null_inds.size:
candles = candles.fillna(method='ffill')
candles.interpolated.iloc[null_inds] = True
# If nulls still in df, drop and check for continuity
if pd.isnull(candles).any().any():
candles = candles.dropna()
continuous = pd.date_range(start=candles.open_date.min(),
end=candles.open_date.max(),
freq = '1H')
if not (continuous == candles.open_date).all():
raise DiscontinuousError(
'''DataFrame doesn't form a continuous date
sequence after interpolation''' )
return candles
def indicators():
for indicator in CustomIndicator.__subclasses__():
yield indicator
# Get timedelta for data acquisition from DB
td = []
num_indicators = 0
for indicator in indicators():
num_indicators+=1
delta = indicator.get_timedelta()
if delta:
td.append(delta)
# HACK? using max(td) will return one candle older than needed
td = max(td)-timedelta(hours=1)
# Get most recent date in candles
if not from_date:
from_date = get_max_from_column(column='open_date')
else:
from_date = tb.DateConvert(from_date).datetime
from_date -= td
if verbose:
print('Fetching data from database...')
# Get raw candles for transformation
candles = Candles().get_raw(from_date = from_date)
candles = interpolate_nulls(candles)
candles.index = candles.symbol
symbols = get_symbols()
num_symbols = len(symbols)
total_iterations = num_symbols*num_indicators
# Calculate indicators
ins = pd.DataFrame()
count = 0
for indicator in indicators():
indicator_data = pd.DataFrame()
for symbol in symbols:
indicator_name = indicator.__name__
sub_candles = candles.loc[symbol]
sub_candles.index = sub_candles.open_date
sub_candles = sub_candles.sort_index()
transformed = pd.DataFrame(indicator()._transform(sub_candles))
transformed['symbol'] = symbol
transformed.columns = [indicator_name, 'symbol']
indicator_data = pd.concat([indicator_data, transformed])
count+=1
if verbose:
tb.progress_bar(
count, total_iterations,
f'Calculating {indicator_name}'
)
# return indicator_data
if ins.empty:
ins = indicator_data
else:
ins = ins.merge(indicator_data, on = ['symbol', 'open_date'])
candles = candles.reset_index(drop=True)
ins = ins.merge(candles, on = ['symbol', 'open_date'])
return ins.dropna()
def test_most_recent_date(self):
# Most recent date should be the most recent candle in the db
max_date = get_max_from_column(column='open_date')
engineered_data = core.engineer_data()
assert engineered_data.open_date.max() == max_date