def calculate_portfolio_for_selected(self):
from portfolios.providers.data.django import DataProviderDjango
from portfolios.providers.execution.django import ExecutionProviderDjango
from portfolios.calculation import Unsatisfiable, calculate_portfolio, get_instruments
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
try:
weights, er, stdev = calculate_portfolio(self.selected_settings,
data_provider,
execution_provider, True,
idata)
portfolio = Portfolio.objects.create(
setting=self.selected_settings,
stdev=stdev,
er=er,
)
items = [
PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.iteritems()
]
PortfolioItem.objects.bulk_create(items)
except Unsatisfiable:
# We detect when loading a goal in the allocation screen if there has been no portfolio created
# and return a message to the user. It it perfectly reasonable for a goal to be created without a
# portfolio.
logger.exception(
"No suitable portfolio could be found. Leaving empty.")
def handle(self, *args, **options):
from portfolios.providers.execution.django import ExecutionProviderDjango
data_provider = DataProviderDjango(timezone.now().date())
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
for gs in GoalSetting.objects.all():
if gs.can_rebalance:
logger.info('Rebalancing goal %s' % gs.goal)
rebalance(gs.goal, idata, data_provider, execution_provider)
def setUp(self):
self.t1 = TickerFactory.create(symbol='SPY', unit_price=5)
self.t2 = TickerFactory.create(symbol='VEA', unit_price=5)
self.t3 = TickerFactory.create(symbol='TIP', unit_price=100)
self.t4 = TickerFactory.create(symbol='IEV', unit_price=100)
self.t5 = TickerFactory.create(symbol='IEV2',
unit_price=100,
asset_class=self.t4.asset_class)
self.equity = AssetFeatureValueFactory.create(
name='equity', assets=[self.t1, self.t2])
self.bond = AssetFeatureValueFactory.create(name='bond',
assets=[self.t3, self.t4])
self.goal_settings = GoalSettingFactory.create()
asset_classes = [
self.t1.asset_class, self.t2.asset_class, self.t3.asset_class,
self.t4.asset_class
]
portfolio_set = PortfolioSetFactory.create(name='set',
risk_free_rate=0.01,
asset_classes=asset_classes)
self.goal = GoalFactory.create(approved_settings=self.goal_settings,
active_settings=self.goal_settings,
cash_balance=100,
portfolio_set=portfolio_set)
self.tickers = [self.t1, self.t2, self.t3, self.t4, self.t4]
self.prices = [4, 4, 90, 90, 95]
self.quantities = [5, 5, 5, 5, 5]
self.executed = [
date(2015, 1, 1),
date(2016, 1, 1),
date(2015, 1, 1),
date(2016, 1, 1),
date(2016, 1, 1)
]
self.execution_details = []
for i in range(5):
execution = Fixture1.create_execution_details(
self.goal, self.tickers[i], self.quantities[i], self.prices[i],
self.executed[i])
self.execution_details.append(execution)
self.data_provider = DataProviderDjango(mocked_now.date())
self.execution_provider = ExecutionProviderDjango()
MarkowitzScaleFactory.create()
self.setup_performance_history()
self.idata = get_instruments(self.data_provider)
self.portfolio = PortfolioFactory.create(setting=self.goal_settings)
self.current_weights = get_held_weights(self.goal)
def process(data_provider, execution_provider, delay):
# actually tickers are created here - we need to set proper asset class for each ticker
goals = Goal.objects.all()
#get_markowitz_scale(self)
#MarkowitzScaleFactory.create()
data_provider.get_goals()
build_instruments(data_provider)
# optimization fails due to
#portfolios_stats = calculate_portfolios(setting=goal.selected_settings,
# data_provider=data_provider,
# execution_provider=execution_provider)
#portfolio_stats = calculate_portfolio(settings=goal.selected_settings,
# data_provider=data_provider,
# execution_provider=execution_provider)
for goal in goals:
weights, instruments, reason = rebalance(
idata=get_instruments(data_provider),
goal=goal,
data_provider=data_provider,
execution_provider=execution_provider)
new_positions = build_positions(goal, weights, instruments)
# create sell requests first
mor, requests = create_request(goal,
new_positions,
reason,
execution_provider=execution_provider,
data_provider=data_provider,
allowed_side=-1)
approve_mor(mor)
# process sells
execute(delay)
for goal in goals:
# now create buys - but only use cash to finance proceeds of buys
mor, requests = create_request(goal,
new_positions,
reason,
execution_provider=execution_provider,
data_provider=data_provider,
allowed_side=1)
approve_mor(mor)
execute(delay)
def setUp(self):
self.t1 = TickerFactory.create(symbol='SPY', unit_price=5)
self.t2 = TickerFactory.create(symbol='VEA', unit_price=5)
self.t3 = TickerFactory.create(symbol='TIP', unit_price=100)
self.t4 = TickerFactory.create(symbol='IEV', unit_price=100)
self.equity = AssetFeatureValueFactory.create(
name='equity', assets=[self.t1, self.t2])
self.bond = AssetFeatureValueFactory.create(name='bond',
assets=[self.t3, self.t4])
self.goal_settings = GoalSettingFactory.create()
asset_classes = [
self.t1.asset_class, self.t2.asset_class, self.t3.asset_class,
self.t4.asset_class
]
portfolio_set = PortfolioSetFactory.create(name='set',
risk_free_rate=0.01,
asset_classes=asset_classes)
self.goal = GoalFactory.create(approved_settings=self.goal_settings,
cash_balance=100,
portfolio_set=portfolio_set)
Fixture1.create_execution_details(self.goal, self.t1, 5, 4,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t2, 5, 4,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t3, 5, 90,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t4, 5, 90,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t4, 5, 90,
date(2016, 1, 1))
self.data_provider = DataProviderDjango()
self.execution_provider = ExecutionProviderDjango()
MarkowitzScaleFactory.create()
self.setup_performance_history()
self.idata = get_instruments(self.data_provider)
def test_calculate_portfolio(self):
goal1 = Fixture1.goal1()
goal1.portfolio_set.asset_classes = [
AssetClass.objects.get(name="US_BONDS"),
AssetClass.objects.get(name="AU_STOCKS"),
AssetClass.objects.get(name="AU_STOCK_MUTUALS")
]
goal1.selected_settings.metric_group.metrics = [GoalMetric.objects.create(group=Fixture1.metric_group1(),
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
rebalance_type="1",
configured_val=0.0,
comparison=2,
rebalance_thr=0.05)
]
goal1.selected_settings.SYSTEM_CURRENCY = 'USD'
goal1.cash_balance = 1000
idata = get_instruments(self._data_provider)
portfolio, er, var = calculate_portfolio(settings=goal1.selected_settings,
data_provider=self._data_provider,
execution_provider=self._execution_provider,
idata=idata)
self.assertEqual(len(portfolio), 4)
def handle(self, *args, **options):
idata = get_instruments(DataProviderDjango())
for goal in Goal.objects.all():
measure(goal, idata)
def test_backtest(self):
setup = TestSetup()
# actually tickers are created here - we need to set proper asset class for each ticker
self.create_goal()
setup.create_goal(self.goal)
setup.data_provider.initialize_tickers()
setup.data_provider.move_date_forward()
backtester = Backtester()
print("backtesting " + str(setup.data_provider.get_current_date()))
build_instruments(setup.data_provider)
# optimization fails due to
portfolios_stats = calculate_portfolios(
setting=setup.goal.selected_settings,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
portfolio_stats = calculate_portfolio(
settings=setup.goal.selected_settings,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
weights, instruments, reason = rebalance(
idata=get_instruments(setup.data_provider),
goal=setup.goal,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
new_positions = build_positions(setup.goal, weights, instruments)
# create sell requests first
mor, requests = create_request(
setup.goal,
new_positions,
reason,
execution_provider=setup.execution_provider,
data_provider=setup.data_provider,
allowed_side=-1)
# process sells
backtester.execute(mor)
# now create buys - but only use cash to finance proceeds of buys
mor, requests = create_request(
setup.goal,
new_positions,
reason,
execution_provider=setup.execution_provider,
data_provider=setup.data_provider,
allowed_side=1)
backtester.execute(mor)
transaction_cost = np.sum([abs(r.volume) for r in requests]) * 0.005
# So, the rebalance could not be in place if the excecution algo might not determine how much it will cost to rebalance.
# this does not work - make it work with Django execution provider - use EtnaOrders
#performance = backtester.calculate_performance(execution_provider=setup.execution_provider)
self.assertTrue(True)
def create_settings(plan):
"""
Creates some settings that can be used to create a real retirement goal if desired.
:param plan: The retirement plan to create settings for.
:return: A GoalSetting object that has been saved in the database.
The caller needs to destroy it if it is no longer required
:raises Unsatisfiable if no suitable portfolio could be found for the plan.
"""
metric_group = GoalMetricGroup.objects.create(type=GoalMetricGroup.TYPE_CUSTOM)
settings = GoalSetting.objects.create(
target=0,
completion=timezone.now().date(),
hedge_fx=False,
metric_group=metric_group,
)
risk_metric = GoalMetric.objects.create(group=metric_group,
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=plan.desired_risk)
# Create a mock goal so we can call calculate_portfolio
class MockGoal(object):
portfolio_set = plan.client.advisor.default_portfolio_set
id = 0
available_balance = 100000
current_balance = 100000
def __str__(self):
return "Retiresmartz calculation Goal for plan: {}".format(plan)
# Create a dummy settings object for the calculation.
# We need to do this because we are using a fake goal, as we don't have a real goal yet.
settings_data = GoalSettingSerializer(instance=settings).data
calc_settings = GoalSettingStatelessSerializer.create_stateless(settings_data, MockGoal())
data_provider = DataProviderDjango()
idata = get_instruments(data_provider)
weights, er, stdev = calculate_portfolio(
settings=calc_settings,
idata=idata,
data_provider=data_provider,
execution_provider=ExecutionProviderDjango()
)
portfolio = Portfolio.objects.create(
setting=settings,
stdev=stdev,
er=er,
)
items = [PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.items()]
PortfolioItem.objects.bulk_create(items)
return settings
def handle(self, *args, **options):
# find extremes
data_provider = DataProviderDjango()
# Get the funds from the instruments table
covars, funds, masks = get_instruments(data_provider)
logger.debug("Using instruments:\n {}\n\n with covars:\n{}".format(
funds, covars))
sigma = covars.values
mu = funds[INSTRUMENT_TABLE_EXPECTED_RETURN_LABEL].values
# Get the instruments with the best BL ER.
perfix = np.argmax(mu)
itms = np.argwhere(mu == mu[perfix])
ilist = [i[0] for i in itms.tolist()]
logger.info(
"Found largest ER instruments: {} at index: {}, ilist: {}".format(
funds.index[itms], itms, ilist))
xs, constraints = get_core_constraints(funds.shape[0])
constraints += [xs >= 0]
# Find the lambda that gives only the best BL ER.
lowerb = 0.0
upperb = 100000000.0
mval = 10
while upperb - lowerb > .001: # We want lambda to 3 decimal places
weights, cost = markowitz_optimizer_3(xs, sigma, mval, mu,
constraints)
changed = False
for ix, weight in enumerate(weights):
# print("ix={}, weight={}".format(ix, weight))
if ix not in itms and weight > MIN_PORTFOLIO_PCT:
lowerb = mval
mval = min(mval * 2, mval + ((upperb - mval) / 2))
changed = True
break
if not changed:
upperb = mval
mval -= ((mval - lowerb) / 2)
max_lambda = round(mval, 3)
logger.debug("Weights at max_lambda: {}".format(weights))
logger.info("Found MAX_LAMBDA: {}".format(max_lambda))
# Find the least variance portfolio.
constraints.append(mul_elemwise(mu, xs) >= 0)
weights, cost = markowitz_optimizer_3(xs, sigma, 0.0, mu, constraints)
# Remove any below minimum percent and round to find the target portfolio
weights[weights < MIN_PORTFOLIO_PCT] = 0
target = np.round(weights, 2)
# Find the lambda that gives the same portfolio as the target.
lowerb = 0.0
upperb = max_lambda
mval = max_lambda / 2
while upperb - lowerb > .001: # We want lambda to 3 decimal places
weights, cost = markowitz_optimizer_3(xs, sigma, mval, mu,
constraints)
weights[weights < MIN_PORTFOLIO_PCT] = 0
comp = np.round(weights, 2)
if np.allclose(target, comp):
lowerb = mval
mval += ((upperb - mval) / 2)
else:
upperb = mval
mval -= ((mval - lowerb) / 2)
min_lambda = round(mval, 3)
logger.info("Found MIN_LAMBDA: {}".format(min_lambda))
vals = get_risk_curve(min_lambda, max_lambda)
data_provider.set_markowitz_scale(dt=now().today(),
mn=min_lambda,
mx=max_lambda,
a=vals[0],
b=vals[1],
c=vals[2])
execution_provider=setup.execution_provider)
'''
# calculate current portfolio stats
portfolios_stats = calculate_portfolios(
setting=setup.goal.active_settings,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
portfolio_stats = calculate_portfolio(
settings=setup.goal.active_settings,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
# generate orders for tomorrow
#try:
requests = rebalance(idata=get_instruments(setup.data_provider),
goal=setup.goal,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
#except:
# print("reblance not succesful")
# requests = [setup.execution_provider.create_empty_market_order()]
backtester.execute_order(settings=setup.goal,
order=requests,
data_provider=setup.data_provider,
execution_provider=setup.execution_provider)
performance = backtester.calculate_performance(
execution_provider=setup.execution_provider)
def process(data_provider, execution_provider, delay, goals=None):
# actually tickers are created here - we need to set proper asset class for each ticker
if goals is None:
goals = Goal.objects.filter(state=Goal.State.ACTIVE.value)
#get_markowitz_scale(self)
#MarkowitzScaleFactory.create()
data_provider.get_goals()
build_instruments(data_provider)
# optimization fails due to
#portfolios_stats = calculate_portfolios(setting=goal.selected_settings,
# data_provider=data_provider,
# execution_provider=execution_provider)
#portfolio_stats = calculate_portfolio(settings=goal.selected_settings,
# data_provider=data_provider,
# execution_provider=execution_provider)
goals_list = []
for goal in goals:
weights, instruments, reason = rebalance(
idata=get_instruments(data_provider),
goal=goal,
data_provider=data_provider,
execution_provider=execution_provider)
new_positions = build_positions(goal, weights, instruments)
goals_list.append({'goal': goal, 'new_positions': new_positions})
if settings.DEBUG:
print('---------------------------------------------------')
print('weights:', weights)
print('instruments:', instruments)
for item in goals_list:
goal = item['goal']
new_positions = item['new_positions']
mor, requests = create_request(goal,
new_positions,
reason,
execution_provider=execution_provider,
data_provider=data_provider,
allowed_side=-1)
if settings.DEBUG:
print('---------------------------------------------------')
print(requests)
print('>>>>>>>>>>>>', 'goal:', 'state->', goal.state, 'account->',
goal.account, 'name->', goal.name, 'portfolio_set->',
goal.portfolio_set, 'cash_balance->', goal.cash_balance)
print('new_positions:', new_positions)
approve_mor(mor)
# process sells
execute(delay)
for item in goals_list:
goal = item['goal']
new_positions = item['new_positions']
# now create buys - but only use cash to finance proceeds of buys
mor, requests = create_request(goal,
new_positions,
reason,
execution_provider=execution_provider,
data_provider=data_provider,
allowed_side=1)
approve_mor(mor)
# process buys
execute(delay)
def create(self, validated_data):
"""
Override the default create because we need to generate a portfolio.
:param validated_data:
:return: The created Goal
"""
account = validated_data['account']
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
with transaction.atomic():
metric_group = GoalMetricGroup.objects.create(
type=GoalMetricGroup.TYPE_CUSTOM)
settings = GoalSetting.objects.create(
target=validated_data['target'],
completion=validated_data['completion'],
hedge_fx=False,
metric_group=metric_group,
)
portfolio_provider_id = validated_data[
'portfolio_provider'] if 'portfolio_provider' in validated_data else get_default_provider_id(
)
portfolio_set_id = ''
portfolio_providers = PortfolioProvider.objects.all()
portfolio_provider = get_default_provider()
for pp in portfolio_providers:
if pp.id == portfolio_provider_id:
portfolio_provider = pp
if portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_KRANE:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_KRANE
elif portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_AON:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_AON
elif portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_LEE:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_LEE
else:
raise Exception('unhandled portfolio_provider_id')
portfolio_sets = PortfolioSet.objects.all()
portfolio_set = account.default_portfolio_set
for ps in portfolio_sets:
if ps.type == portfolio_set_type:
portfolio_set = ps
goal = Goal.objects.create(
account=account,
name=validated_data['name'],
type=validated_data['type'],
portfolio_set=portfolio_set,
portfolio_provider=portfolio_provider,
selected_settings=settings,
)
# Based on the risk profile, and whether an ethical profile was specified on creation, set up Metrics.
recommended_risk = recommend_risk(settings)
GoalMetric.objects.create(
group=metric_group,
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=recommended_risk)
if validated_data['ethical']:
GoalMetric.objects.create(
group=metric_group,
type=GoalMetric.METRIC_TYPE_PORTFOLIO_MIX,
feature=AssetFeatureValue.Standard.SRI_OTHER.get_object(),
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=1 # Start with 100% ethical.
)
# Make sure the risk score assigned is appropriate for the goal.
try:
validate_risk_score(settings)
except CVE as verr:
raise ValidationError(verr.message)
# Add the initial deposit if specified.
initial_dep = validated_data.pop('initial_deposit', None)
if initial_dep is not None:
Transaction.objects.create(reason=Transaction.REASON_DEPOSIT,
to_goal=goal,
amount=initial_dep)
# Calculate the optimised portfolio
try:
weights, er, stdev = calculate_portfolio(
settings, data_provider, execution_provider, True, idata)
portfolio = Portfolio.objects.create(
setting=settings,
stdev=stdev,
er=er,
)
items = [
PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.iteritems()
]
PortfolioItem.objects.bulk_create(items)
except Unsatisfiable:
# We detect when loading a goal in the allocation screen if there has been no portfolio created
# and return a message to the user. It it perfectly reasonable for a goal to be created without a
# portfolio.
logger.exception(
"No suitable portfolio could be found. Leaving empty.")
return goal
