def cache_series_release_data(series_name): logger.info('Caching derived series release data - %s' % series_name) data = get_series_all_release(series_name) if data is not None: ans = [] release = calculate_first_release(data) if release is not None: release.name = series_name + '|release' ans.append(release) release = calculate_extended_first_release(data) if release is not None: release.name = series_name + '|extendedrelease' ans.append(release) change = calculate_change(data, 5) if change is not None: change.name = series_name + '|change' ans.append(change) change = calculate_change(data, 365) if change is not None: change.name = series_name + '|annualchange' ans.append(change) revision = calculate_revision(data) if revision is not None: revision = tu.resample(revision, release, carry_forward=False).fillna(0.) revision.name = series_name + '|revision' ans.append(revision) if len(ans) > 0: ans = pd.concat(ans, axis=1) logger.info('Storing derived series release data - %s' % series_name) tu.store_timeseries(ans, DATABASE_NAME, CACHE_TABLE_NAME)
def load_asset_prices(self): logger.info('Loading asset prices') self.asset_prices = self.asset_data_loader(self.assets, start_date=self._load_start, end_date=self._load_end) p = tu.resample(self.asset_prices, self.timeline) self._asset_returns = p.diff()
def load_series_all_releases(series_name): logger.info('Loadind series all release - %s' % series_name) try: return _api.get_series_all_releases(series_name) except Exception as e: logger.warn('Failed to get series all releases: %s' % str(e)) return None
def load_series_info(series_name): logger.info('Loading series info - %s' % series_name) try: return _api.get_series_info(series_name) except Exception as e: logger.warn('Failed to get series info: %s' % str(e)) return None
def run_gradient_descent(self): w = self._w obj, obj_prime, stock_weights, step = self.calc_iteration(w) count = 0 ws = [w.copy()] ss = [stock_weights.copy()] objs = [obj] obj_primes = [obj_prime] steps = [step] while count < self.max_iteration and step > self.epsilon: w -= self.speed * obj_prime obj, obj_prime, stock_weights, step = self.calc_iteration(w) count += 1 ws.append(w.copy()) ss.append(stock_weights.copy()) objs.append(obj) obj_primes.append(obj_prime) steps.append(step) if self.logging: logger.info('Count: %d Step: %.2f' % (count, step)) self.ws = pd.concat(ws, axis=1).T self.ss = pd.concat(ss, axis=1).T self.objs = pd.Series(objs, name='Objective') self.obj_primes = pd.concat(obj_primes, axis=1).T self.steps = pd.Series(steps, name='Step') self.ws.index = self.objs.index self.ss.index = self.objs.index self.obj_primes.index = self.objs.index
def run_sim(self): logger.info('Running simulation %s' % self.simulation_name) self.get_timeline() self.load_asset_prices() self.load_signal() self.run_simulation() self.calculate_returns() self.get_analytics()
def download_and_store_series(series_name, series_id, table_name=QUANDL_FUTURES): logger.info('Downloading quandl series %s %s' % (series_name, series_id)) series = load_series(series_id) if series is not None: logger.info('Storing quandl series - %s' % series_name) tu.store_timeseries(series, DATABASE_NAME, table_name, series_name)
def calculate_returns(self): logger.info('Simulating strategy returns') self.asset_returns = self.asset_prices.resample( 'B').last().ffill().diff() start_date = self.start_date if self.start_date > self.positions.first_valid_index( ) else self.positions.first_valid_index() strategy_returns = [] for c in self.positions.columns: rtn = tu.resample( self.positions[c], self.asset_returns).shift() * self.asset_returns.iloc[:, 0] rtn.name = c strategy_returns.append(rtn) self.strategy_returns = pd.concat(strategy_returns, axis=1)[start_date:] self.asset_returns = self.asset_returns[start_date:]
def run_cross_validation(self): if self._y is not None: y = self._y[~self._y.isnull()] x = self._x[~self._y.isnull()] self._seq = get_cross_validation_buckets(len(y), self.cross_validation_buckets) self.errors = [] for i, s in enumerate(self._seq): logger.info('Cross validation bucket %s' % (i + 1)) x_out = x.iloc[s] y_out = y.iloc[s] x_in = x.loc[~x.index.isin(x_out.index)] y_in = y.loc[~y.index.isin(y_out.index)] m = self.model_function(x=x_in, y=y_in, **self.params) pred = self.prediction_function(x=x_out, ans=m) self.errors.append(StumpError(pred.values.flatten(), y_out.values.flatten(), 0.)) self.error_rate = np.mean(self.errors) else: self.errors = None self.error_rate = None
def get_analytics(self): logger.info('Calculating analytics') rtn = pd.concat([self.asset_returns, self.strategy_returns], axis=1) self.analytics = get_returns_analytics(rtn)
def load_signal(self): logger.info('Loading signal') signal = self.signal_loader(start_date=self._load_start, end_date=self._load_end) self.signal = tu.resample(signal, self.timeline)
def get_timeline(self): logger.info('Creating time line') self.timeline = get_timeline(self.start_date, self.end_date, self.data_frequency) self._load_start = self.timeline.index[0] self._load_end = self.timeline.index[-1]
def store_series_all_releases(data, series_name): logger.info('Storing series all release - %s' % series_name) release_bulk_insert(data, RELEASE_TABLE_NAME, series_name)
def store_series(data, series_name): logger.info('Storing series - %s' % series_name) series = data.copy() series.name = series_name du.pandas_bulk_insert(series, DATABASE_NAME, SERIES_TABLE_NAME, du.TIMESERIES_COLUMN_NAME, du.TIMESERIES_INDEX_NAME, du.TIMESERIES_VALUE_NAME)
def store_series_info(data, series_name): logger.info('Storing series info - %s' % series_name) series = data.copy().apply(_encode_string) series.name = series_name du.pandas_bulk_insert(series, DATABASE_NAME, INFO_TABLE_NAME, 'series_name', 'description', 'value')
def estimate_model(self): if self.model is None and self._y is not None: logger.info('Estimating model with %d observations' % len(self._x)) self.model = self.model_function(x=self._x.loc[~self._y.isnull(), :], y=self._y[~self._y.isnull()], **self.params)