def cache_series_release_data(series_name):
logger.info('Caching derived series release data - %s' % series_name)
data = get_series_all_release(series_name)
if data is not None:
ans = []
release = calculate_first_release(data)
if release is not None:
release.name = series_name + '|release'
ans.append(release)
release = calculate_extended_first_release(data)
if release is not None:
release.name = series_name + '|extendedrelease'
ans.append(release)
change = calculate_change(data, 5)
if change is not None:
change.name = series_name + '|change'
ans.append(change)
change = calculate_change(data, 365)
if change is not None:
change.name = series_name + '|annualchange'
ans.append(change)
revision = calculate_revision(data)
if revision is not None:
revision = tu.resample(revision, release, carry_forward=False).fillna(0.)
revision.name = series_name + '|revision'
ans.append(revision)
if len(ans) > 0:
ans = pd.concat(ans, axis=1)
logger.info('Storing derived series release data - %s' % series_name)
tu.store_timeseries(ans, DATABASE_NAME, CACHE_TABLE_NAME)
def load_asset_prices(self):
logger.info('Loading asset prices')
self.asset_prices = self.asset_data_loader(self.assets,
start_date=self._load_start,
end_date=self._load_end)
p = tu.resample(self.asset_prices, self.timeline)
self._asset_returns = p.diff()
def load_series_all_releases(series_name):
logger.info('Loadind series all release - %s' % series_name)
try:
return _api.get_series_all_releases(series_name)
except Exception as e:
logger.warn('Failed to get series all releases: %s' % str(e))
return None
def load_series_info(series_name):
logger.info('Loading series info - %s' % series_name)
try:
return _api.get_series_info(series_name)
except Exception as e:
logger.warn('Failed to get series info: %s' % str(e))
return None
def run_gradient_descent(self):
w = self._w
obj, obj_prime, stock_weights, step = self.calc_iteration(w)
count = 0
ws = [w.copy()]
ss = [stock_weights.copy()]
objs = [obj]
obj_primes = [obj_prime]
steps = [step]
while count < self.max_iteration and step > self.epsilon:
w -= self.speed * obj_prime
obj, obj_prime, stock_weights, step = self.calc_iteration(w)
count += 1
ws.append(w.copy())
ss.append(stock_weights.copy())
objs.append(obj)
obj_primes.append(obj_prime)
steps.append(step)
if self.logging:
logger.info('Count: %d Step: %.2f' % (count, step))
self.ws = pd.concat(ws, axis=1).T
self.ss = pd.concat(ss, axis=1).T
self.objs = pd.Series(objs, name='Objective')
self.obj_primes = pd.concat(obj_primes, axis=1).T
self.steps = pd.Series(steps, name='Step')
self.ws.index = self.objs.index
self.ss.index = self.objs.index
self.obj_primes.index = self.objs.index
def run_sim(self):
logger.info('Running simulation %s' % self.simulation_name)
self.get_timeline()
self.load_asset_prices()
self.load_signal()
self.run_simulation()
self.calculate_returns()
self.get_analytics()
def download_and_store_series(series_name,
series_id,
table_name=QUANDL_FUTURES):
logger.info('Downloading quandl series %s %s' % (series_name, series_id))
series = load_series(series_id)
if series is not None:
logger.info('Storing quandl series - %s' % series_name)
tu.store_timeseries(series, DATABASE_NAME, table_name, series_name)
def calculate_returns(self):
logger.info('Simulating strategy returns')
self.asset_returns = self.asset_prices.resample(
'B').last().ffill().diff()
start_date = self.start_date if self.start_date > self.positions.first_valid_index(
) else self.positions.first_valid_index()
strategy_returns = []
for c in self.positions.columns:
rtn = tu.resample(
self.positions[c],
self.asset_returns).shift() * self.asset_returns.iloc[:, 0]
rtn.name = c
strategy_returns.append(rtn)
self.strategy_returns = pd.concat(strategy_returns,
axis=1)[start_date:]
self.asset_returns = self.asset_returns[start_date:]
def run_cross_validation(self):
if self._y is not None:
y = self._y[~self._y.isnull()]
x = self._x[~self._y.isnull()]
self._seq = get_cross_validation_buckets(len(y), self.cross_validation_buckets)
self.errors = []
for i, s in enumerate(self._seq):
logger.info('Cross validation bucket %s' % (i + 1))
x_out = x.iloc[s]
y_out = y.iloc[s]
x_in = x.loc[~x.index.isin(x_out.index)]
y_in = y.loc[~y.index.isin(y_out.index)]
m = self.model_function(x=x_in, y=y_in, **self.params)
pred = self.prediction_function(x=x_out, ans=m)
self.errors.append(StumpError(pred.values.flatten(), y_out.values.flatten(), 0.))
self.error_rate = np.mean(self.errors)
else:
self.errors = None
self.error_rate = None
def get_analytics(self):
logger.info('Calculating analytics')
rtn = pd.concat([self.asset_returns, self.strategy_returns], axis=1)
self.analytics = get_returns_analytics(rtn)
def load_signal(self):
logger.info('Loading signal')
signal = self.signal_loader(start_date=self._load_start,
end_date=self._load_end)
self.signal = tu.resample(signal, self.timeline)
def get_timeline(self):
logger.info('Creating time line')
self.timeline = get_timeline(self.start_date, self.end_date,
self.data_frequency)
self._load_start = self.timeline.index[0]
self._load_end = self.timeline.index[-1]
def store_series_all_releases(data, series_name):
logger.info('Storing series all release - %s' % series_name)
release_bulk_insert(data, RELEASE_TABLE_NAME, series_name)
def store_series(data, series_name):
logger.info('Storing series - %s' % series_name)
series = data.copy()
series.name = series_name
du.pandas_bulk_insert(series, DATABASE_NAME, SERIES_TABLE_NAME, du.TIMESERIES_COLUMN_NAME, du.TIMESERIES_INDEX_NAME, du.TIMESERIES_VALUE_NAME)
def store_series_info(data, series_name):
logger.info('Storing series info - %s' % series_name)
series = data.copy().apply(_encode_string)
series.name = series_name
du.pandas_bulk_insert(series, DATABASE_NAME, INFO_TABLE_NAME, 'series_name', 'description', 'value')
def estimate_model(self):
if self.model is None and self._y is not None:
logger.info('Estimating model with %d observations' % len(self._x))
self.model = self.model_function(x=self._x.loc[~self._y.isnull(), :], y=self._y[~self._y.isnull()], **self.params)