def calculate_portfolio_for_selected(self):
from portfolios.providers.data.django import DataProviderDjango
from portfolios.providers.execution.django import ExecutionProviderDjango
from portfolios.calculation import Unsatisfiable, calculate_portfolio, get_instruments
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
try:
weights, er, stdev = calculate_portfolio(self.selected_settings,
data_provider,
execution_provider, True,
idata)
portfolio = Portfolio.objects.create(
setting=self.selected_settings,
stdev=stdev,
er=er,
)
items = [
PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.iteritems()
]
PortfolioItem.objects.bulk_create(items)
except Unsatisfiable:
# We detect when loading a goal in the allocation screen if there has been no portfolio created
# and return a message to the user. It it perfectly reasonable for a goal to be created without a
# portfolio.
logger.exception(
"No suitable portfolio could be found. Leaving empty.")
def setUp(self):
self.data_provider = DataProviderDjango()
self.data_provider.get_current_date = MagicMock(
return_value=date(2016, 2, 1))
self.last_cycle_start = date(2016, 1, 5)
self.last_cycle_end = date(
2016, 1, 14) # The last day before the observed next cycle
self.predictor = InvestmentClock(self.data_provider)
def handle(self, *args, **options):
from portfolios.providers.execution.django import ExecutionProviderDjango
data_provider = DataProviderDjango(timezone.now().date())
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
for gs in GoalSetting.objects.all():
if gs.can_rebalance:
logger.info('Rebalancing goal %s' % gs.goal)
rebalance(gs.goal, idata, data_provider, execution_provider)
def handle_noargs(self, **options):
'''
runs process(data_provider, execution_provider, delay) in betasmartz / execution / end_of_day.py
'''
from datetime import datetime
from execution.end_of_day import process
from portfolios.providers.execution.django import ExecutionProviderDjango
from portfolios.providers.data.django import DataProviderDjango
data_provider = DataProviderDjango(datetime.now().date())
execution_provider = ExecutionProviderDjango()
process(data_provider, execution_provider, 5)
def setUp(self):
self.t1 = TickerFactory.create(symbol='SPY', unit_price=5)
self.t2 = TickerFactory.create(symbol='VEA', unit_price=5)
self.t3 = TickerFactory.create(symbol='TIP', unit_price=100)
self.t4 = TickerFactory.create(symbol='IEV', unit_price=100)
self.t5 = TickerFactory.create(symbol='IEV2',
unit_price=100,
asset_class=self.t4.asset_class)
self.equity = AssetFeatureValueFactory.create(
name='equity', assets=[self.t1, self.t2])
self.bond = AssetFeatureValueFactory.create(name='bond',
assets=[self.t3, self.t4])
self.goal_settings = GoalSettingFactory.create()
asset_classes = [
self.t1.asset_class, self.t2.asset_class, self.t3.asset_class,
self.t4.asset_class
]
portfolio_set = PortfolioSetFactory.create(name='set',
risk_free_rate=0.01,
asset_classes=asset_classes)
self.goal = GoalFactory.create(approved_settings=self.goal_settings,
active_settings=self.goal_settings,
cash_balance=100,
portfolio_set=portfolio_set)
self.tickers = [self.t1, self.t2, self.t3, self.t4, self.t4]
self.prices = [4, 4, 90, 90, 95]
self.quantities = [5, 5, 5, 5, 5]
self.executed = [
date(2015, 1, 1),
date(2016, 1, 1),
date(2015, 1, 1),
date(2016, 1, 1),
date(2016, 1, 1)
]
self.execution_details = []
for i in range(5):
execution = Fixture1.create_execution_details(
self.goal, self.tickers[i], self.quantities[i], self.prices[i],
self.executed[i])
self.execution_details.append(execution)
self.data_provider = DataProviderDjango(mocked_now.date())
self.execution_provider = ExecutionProviderDjango()
MarkowitzScaleFactory.create()
self.setup_performance_history()
self.idata = get_instruments(self.data_provider)
self.portfolio = PortfolioFactory.create(setting=self.goal_settings)
self.current_weights = get_held_weights(self.goal)
def handle(self, *args, **options):
# calculate portfolios
data_provider = DataProviderDjango()
exec_provider = ExecutionProviderDjango()
for goal in Goal.objects.all():
if goal.selected_settings is not None:
try:
calculate_portfolios(setting=goal.selected_settings,
data_provider=data_provider,
execution_provider=exec_provider)
except Unsatisfiable as e:
logger.warn(e)
def get_risk_score(goal, weights, idata, data_provider, execution_provider):
"""
Returns the risk score for the provided weights.
:param goal:
:param weights:
:param idata:
:return:
"""
if len(weights) == 0:
logger.info(
"No holdings for goal: {} so current risk score is 0.".format(
goal))
return 0.0
# Get the cost of a clean optimisation using the current constraints.
current_cost = optimize_settings(goal.approved_settings, idata,
data_provider, execution_provider)[1]
# Set up the required data
covars, instruments, masks = idata
instruments['_weight_'] = 0.0
# Build the indexes from the passed in weights
goal_symbol_ixs = []
for id, weight in weights.items():
ticker = data_provider.get_ticker(id)
goal_symbol_ixs.append(instruments.index.get_loc(ticker.symbol))
instruments.loc[ticker.symbol, '_weight_'] = weight
goal_instruments = instruments.iloc[goal_symbol_ixs]
samples = data_provider.get_instrument_daily_prices(ticker).count()
mu, sigma = run_bl(instruments, covars, goal_instruments, samples,
goal.portfolio_set)
#ws = np.expand_dims(goal_instruments['_weight_'].values, 0)
'''
BVE - 2017/04/19 - line above produces an array with a different dimension to mu and sigma
... this doesn't make sense, and produces an error, so I have replaced this by the next line,
which makes more sense ...
'''
ws = np.expand_dims(instruments['_weight_'].values, 0)
# Get the lambda that produces the same cost as the optimum portfolio using the configured risk score.
# TODO: I can probably do this algebraically, and not have to run an iterative minimizer, but this is easy for now.
def f(lam):
cost = markowitz_cost(ws, sigma, lam, mu)
return abs(current_cost - cost)
res = minimize_scalar(f)
if math.isnan(res.x):
raise Exception("Couldn't find appropriate lambda")
logger.debug("Calculated lambda: {} for goal: {}".format(res.x, goal))
return lambda_to_risk_score(lam=res.x, data_provider=DataProviderDjango())
def calculate_all_portfolios(self, request, pk=None, **kwargs):
"""
Called to calculate all the portfolio objects for 100 different risk scores for the supplied settings.
"""
goal = self.get_object()
check_state(Goal.State(goal.state), Goal.State.ACTIVE)
setting_str = request.query_params.get('setting', None)
if not setting_str:
logger.debug(
'setting parameter missing from calculate_all_portfolios query'
)
raise ValidationError(
"Query parameter 'setting' must be specified and a valid JSON string"
)
try:
setting = ujson.loads(setting_str)
except ValueError:
logger.debug(
'setting parameter for calculate_all_portfolios query not valid json'
)
raise ValidationError(
"Query parameter 'setting' must be a valid json string")
# Create the settings from the dict
serializer = serializers.GoalSettingStatelessSerializer(data=setting)
serializer.is_valid(raise_exception=True)
settings = serializer.create_stateless(serializer.validated_data, goal)
# Calculate the portfolio
try:
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
data = [
self.build_portfolio_data(item[1], item[0]) for item in
calculate_portfolios(setting=settings,
data_provider=data_provider,
execution_provider=execution_provider)
]
return Response(data)
except Unsatisfiable as e:
rdata = {'reason': "No portfolio could be found: {}".format(e)}
if e.req_funds is not None:
rdata['req_funds'] = e.req_funds
return Response({'error': rdata},
status=status.HTTP_400_BAD_REQUEST)
def process(modeladmin, request, queryset):
from datetime import datetime
from execution.end_of_day import process
from portfolios.providers.execution.django import ExecutionProviderDjango
from portfolios.providers.data.django import DataProviderDjango
data_provider = DataProviderDjango(datetime.now().date())
execution_provider = ExecutionProviderDjango()
try:
process(data_provider, execution_provider, 5, queryset)
modeladmin.message_user(
request, "Selected Goals has been processed successfully.")
except Exception as e:
modeladmin.message_user(request,
"Error! Can't process.",
level=messages.ERROR)
def test_calculate_portfolio(self):
# TODO
# constraints -> limit them -> maximum minimum value of 5%, maximum max value of 95%
asset_class1 = AssetClassFactory.create(name='US_TOTAL_BOND_MARKET')
asset_class2 = AssetClassFactory.create(name='HEDGE_FUNDS')
fund0 = TickerFactory.create(symbol='IAGG', asset_class=asset_class1)
fund0 = TickerFactory.create(symbol='GRFXX', asset_class=asset_class1)
fund1 = TickerFactory.create(symbol='ITOT', asset_class=asset_class2)
fund0 = TickerFactory.create(symbol='IPO')
fund0 = TickerFactory.create(symbol='AGG', asset_class=asset_class1)
fund6 = TickerFactory.create(symbol='rest')
ps1 = PortfolioSetFactory \
.create(asset_classes=[asset_class1, asset_class2, fund6.asset_class])
feature = AssetFeatureValueFactory.create()
feature.assets.add(fund6)
settings = GoalSettingFactory.create()
risk_metric = GoalMetricFactory.create(
group=settings.metric_group,
type=GoalMetric.METRIC_TYPE_RISK_SCORE)
mix_metric = GoalMetricFactory.create(
group=settings.metric_group,
type=GoalMetric.METRIC_TYPE_PORTFOLIO_MIX,
feature=feature,
comparison=GoalMetric.METRIC_COMPARISON_MINIMUM,
configured_val=0.5)
goal = GoalFactory.create(selected_settings=settings,
portfolio_set=ps1)
# Create some instrument data for the two assets
self.m_scale = MarkowitzScaleFactory.create()
populate_prices(500, asof=mocked_now.date())
populate_cycle_obs(500, asof=mocked_now.date())
populate_cycle_prediction(asof=mocked_now.date())
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = build_instruments(data_provider)
result = calculate_portfolio(settings=settings,
data_provider=data_provider,
execution_provider=execution_provider,
idata=idata)
self.assertTrue(True)
def test_calculate_portfolio_old(self):
fund0 = TickerFactory.create(symbol='IAGG')
fund1 = TickerFactory.create(symbol='ITOT')
fund5 = TickerFactory.create(symbol='GRFXX')
fund2 = TickerFactory.create(symbol='VEA')
fund0 = TickerFactory.create(symbol='IPO')
fund3 = TickerFactory.create(symbol='EEM')
fund4 = TickerFactory.create(symbol='AGG')
AssetFeatureValueFactory.create(
assets=[fund1, fund2, fund3, fund4, fund5])
ps1 = PortfolioSetFactory \
.create(asset_classes=[fund1.asset_class, fund2.asset_class, fund3.asset_class, fund4.asset_class, fund5.asset_class])
# Create a settings object with a metric for a feature with no instruments in the current portfolio set.
feature = AssetFeatureValueFactory.create()
settings = GoalSettingFactory.create()
risk_metric = GoalMetricFactory.create(group=settings.metric_group)
mix_metric = GoalMetricFactory.create(
group=settings.metric_group,
type=GoalMetric.METRIC_TYPE_PORTFOLIO_MIX,
feature=feature,
comparison=GoalMetric.METRIC_COMPARISON_MAXIMUM,
configured_val=.3)
goal = GoalFactory.create(selected_settings=settings,
portfolio_set=ps1)
# The below fund has the desired feature, but is not in the goal's portfolio set.
feature.assets.add(fund1)
# Create some instrument data for the two assets
self.m_scale = MarkowitzScaleFactory.create()
# populate the data needed for the prediction
# We need at least 500 days as the cycles go up to 70 days and we need at least 7 cycles.
populate_prices(500, asof=mocked_now.date())
populate_cycle_obs(500, asof=mocked_now.date())
populate_cycle_prediction(asof=mocked_now.date())
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = build_instruments(data_provider)
result = calculate_portfolio_old(settings=settings,
data_provider=data_provider,
execution_provider=execution_provider,
idata=idata)
self.assertTrue(True)
def calculate_performance(self, request, pk=None, **kwargs):
port_items_str = request.query_params.get('items', None)
errstr = "Query parameter 'items' must be specified and a valid JSON string [[asset_id, weight], ...]"
if not port_items_str:
raise ValidationError(errstr)
try:
port_items = ujson.loads(port_items_str)
except ValueError:
raise ValidationError(errstr)
total_weight = sum([item[1] for item in port_items])
if total_weight > 1.0001:
raise ValidationError(
"Sum of item weights must be less than or equal to 1")
try:
er, stdev, _ = current_stats_from_weights(port_items,
DataProviderDjango())
except Unsatisfiable as e:
raise ValidationError(e.msg)
return Response({'er': er, 'stdev': stdev})
def setUp(self):
self.t1 = TickerFactory.create(symbol='SPY', unit_price=5)
self.t2 = TickerFactory.create(symbol='VEA', unit_price=5)
self.t3 = TickerFactory.create(symbol='TIP', unit_price=100)
self.t4 = TickerFactory.create(symbol='IEV', unit_price=100)
self.equity = AssetFeatureValueFactory.create(
name='equity', assets=[self.t1, self.t2])
self.bond = AssetFeatureValueFactory.create(name='bond',
assets=[self.t3, self.t4])
self.goal_settings = GoalSettingFactory.create()
asset_classes = [
self.t1.asset_class, self.t2.asset_class, self.t3.asset_class,
self.t4.asset_class
]
portfolio_set = PortfolioSetFactory.create(name='set',
risk_free_rate=0.01,
asset_classes=asset_classes)
self.goal = GoalFactory.create(approved_settings=self.goal_settings,
cash_balance=100,
portfolio_set=portfolio_set)
Fixture1.create_execution_details(self.goal, self.t1, 5, 4,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t2, 5, 4,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t3, 5, 90,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t4, 5, 90,
date(2016, 1, 1))
Fixture1.create_execution_details(self.goal, self.t4, 5, 90,
date(2016, 1, 1))
self.data_provider = DataProviderDjango()
self.execution_provider = ExecutionProviderDjango()
MarkowitzScaleFactory.create()
self.setup_performance_history()
self.idata = get_instruments(self.data_provider)
def calculate_portfolio(self, request, pk=None, **kwargs):
"""
Called to calculate a portfolio object for a set of supplied settings.
"""
goal = self.get_object()
check_state(Goal.State(goal.state),
[Goal.State.ACTIVE, Goal.State.INACTIVE])
setting_str = request.query_params.get('setting', None)
if not setting_str:
raise ValidationError(
"Query parameter 'setting' must be specified and a valid JSON string"
)
try:
setting = ujson.loads(setting_str)
except ValueError:
raise ValidationError(
"Query parameter 'setting' must be a valid json string")
# Create the settings object from the dict
serializer = serializers.GoalSettingStatelessSerializer(data=setting)
serializer.is_valid(raise_exception=True)
settings = serializer.create_stateless(serializer.validated_data, goal)
try:
data = self.build_portfolio_data(
calculate_portfolio(
settings=settings,
data_provider=DataProviderDjango(),
execution_provider=ExecutionProviderDjango()))
return Response(data)
except Unsatisfiable as e:
rdata = {'reason': "No portfolio could be found: {}".format(e)}
if e.req_funds is not None:
rdata['req_funds'] = e.req_funds
return Response({'error': rdata},
status=status.HTTP_400_BAD_REQUEST)
def create(self, validated_data):
"""
Puts the passed settings into the 'selected_settings' field on the passed goal.
"""
goal = validated_data.pop('goal')
# MEtric group and hedge_fx are required on the model. I have no idea why I have to check it again here.
metrics_data = validated_data.pop('metric_group', None)
if metrics_data is None:
raise ValidationError({"metric_group": "is required"})
hedge_fx = validated_data.pop('hedge_fx', None)
if hedge_fx is None:
raise ValidationError({"hedge_fx": "is required"})
tx_data = validated_data.pop('recurring_transactions', None)
port_data = validated_data.pop('portfolio', None)
with transaction.atomic():
gid = metrics_data.get('id', None)
if gid is None:
metric_group = GoalMetricGroup.objects.create()
metrics = metrics_data.get('metrics')
mo = []
for i_data in metrics:
if 'measured_val' in i_data:
raise ValidationError({"measured_val": "is read-only"})
mo.append(
GoalMetric(
group=metric_group,
type=i_data['type'],
feature=i_data.get('feature', None),
comparison=i_data['comparison'],
rebalance_type=i_data['rebalance_type'],
rebalance_thr=i_data['rebalance_thr'],
configured_val=i_data['configured_val'],
))
GoalMetric.objects.bulk_create(mo)
else:
metric_group = GoalMetricGroup.objects.get(gid)
setting = GoalSetting.objects.create(
metric_group=metric_group,
# Target not required, so use default from model if omitted.
target=validated_data.get('target', 0),
completion=validated_data['completion'],
hedge_fx=hedge_fx,
rebalance=validated_data.get('rebalance', True),
)
# Get the current portfolio statistics of the given weights if specified.
if port_data is not None:
port_items_data = port_data.pop('items')
try:
er, stdev, idatas = current_stats_from_weights(
[(item['asset'].id, item['weight'])
for item in port_items_data], DataProviderDjango())
except Unsatisfiable as e:
raise ValidationError(e.msg)
port = Portfolio.objects.create(setting=setting,
er=er,
stdev=stdev)
PortfolioItem.objects.bulk_create([
PortfolioItem(portfolio=port,
asset=i_data['asset'],
weight=i_data['weight'],
volatility=idatas[i_data['asset'].id])
for i_data in port_items_data
])
if tx_data is not None:
RecurringTransaction.objects.bulk_create([
RecurringTransaction(setting=setting,
schedule=i_data['schedule'],
enabled=i_data['enabled'],
begin_date=i_data['begin_date'],
growth=i_data['growth'],
amount=i_data['amount'],
account_id=i_data['account_id'])
for i_data in tx_data
])
try:
goal.set_selected(setting)
except CVE as verr:
raise ValidationError(verr.message)
return setting
def handle(self, *args, **options):
# find extremes
data_provider = DataProviderDjango()
# Get the funds from the instruments table
covars, funds, masks = get_instruments(data_provider)
logger.debug("Using instruments:\n {}\n\n with covars:\n{}".format(
funds, covars))
sigma = covars.values
mu = funds[INSTRUMENT_TABLE_EXPECTED_RETURN_LABEL].values
# Get the instruments with the best BL ER.
perfix = np.argmax(mu)
itms = np.argwhere(mu == mu[perfix])
ilist = [i[0] for i in itms.tolist()]
logger.info(
"Found largest ER instruments: {} at index: {}, ilist: {}".format(
funds.index[itms], itms, ilist))
xs, constraints = get_core_constraints(funds.shape[0])
constraints += [xs >= 0]
# Find the lambda that gives only the best BL ER.
lowerb = 0.0
upperb = 100000000.0
mval = 10
while upperb - lowerb > .001: # We want lambda to 3 decimal places
weights, cost = markowitz_optimizer_3(xs, sigma, mval, mu,
constraints)
changed = False
for ix, weight in enumerate(weights):
# print("ix={}, weight={}".format(ix, weight))
if ix not in itms and weight > MIN_PORTFOLIO_PCT:
lowerb = mval
mval = min(mval * 2, mval + ((upperb - mval) / 2))
changed = True
break
if not changed:
upperb = mval
mval -= ((mval - lowerb) / 2)
max_lambda = round(mval, 3)
logger.debug("Weights at max_lambda: {}".format(weights))
logger.info("Found MAX_LAMBDA: {}".format(max_lambda))
# Find the least variance portfolio.
constraints.append(mul_elemwise(mu, xs) >= 0)
weights, cost = markowitz_optimizer_3(xs, sigma, 0.0, mu, constraints)
# Remove any below minimum percent and round to find the target portfolio
weights[weights < MIN_PORTFOLIO_PCT] = 0
target = np.round(weights, 2)
# Find the lambda that gives the same portfolio as the target.
lowerb = 0.0
upperb = max_lambda
mval = max_lambda / 2
while upperb - lowerb > .001: # We want lambda to 3 decimal places
weights, cost = markowitz_optimizer_3(xs, sigma, mval, mu,
constraints)
weights[weights < MIN_PORTFOLIO_PCT] = 0
comp = np.round(weights, 2)
if np.allclose(target, comp):
lowerb = mval
mval += ((upperb - mval) / 2)
else:
upperb = mval
mval -= ((mval - lowerb) / 2)
min_lambda = round(mval, 3)
logger.info("Found MIN_LAMBDA: {}".format(min_lambda))
vals = get_risk_curve(min_lambda, max_lambda)
data_provider.set_markowitz_scale(dt=now().today(),
mn=min_lambda,
mx=max_lambda,
a=vals[0],
b=vals[1],
c=vals[2])
def handle(self, *args, **options):
idata = get_instruments(DataProviderDjango())
for goal in Goal.objects.all():
measure(goal, idata)
def update(self, setting, validated_data):
"""
Overwrite update to deal with nested writes.
:param setting: The item we're updating
:param validated_data:
:return:
"""
goal = validated_data.pop('goal')
metrics_data = validated_data.pop('metric_group', None)
tx_data = validated_data.pop('recurring_transactions', None)
port_data = validated_data.pop('portfolio', None)
with transaction.atomic():
# Create a new setting.
old_setting = setting
setting = copy.copy(setting)
setting.pk = None
for attr, value in validated_data.items():
if attr not in [
'target', 'completion', 'hedge_fx', 'rebalance'
]:
raise ValidationError({
"msg":
"{} is not a valid field for updating a goal setting.".
format(attr)
})
setattr(setting, attr, value)
if metrics_data is not None:
gid = metrics_data.get('id', None)
if gid is None:
group = GoalMetricGroup.objects.create()
setting.metric_group = group
metrics = metrics_data.get('metrics')
mo = []
for i_data in metrics:
if 'measured_val' in i_data:
raise ValidationError(
{"msg": "measured_val is read-only"})
mo.append(
GoalMetric(
group=group,
type=i_data['type'],
feature=i_data.get('feature', None),
comparison=i_data['comparison'],
rebalance_type=i_data['rebalance_type'],
rebalance_thr=i_data['rebalance_thr'],
configured_val=i_data['configured_val'],
))
GoalMetric.objects.bulk_create(mo)
else:
setting.metric_group = GoalMetricGroup.objects.get(gid)
# Otherwise we just use the old settings object metric group that will be copied in the above copy()
setting.save()
# Do the portfolio.
if port_data is None:
# We can't just change the setting object the portfolio is pointing at as the old setting may still be
# active as an approved setting. WE need to create a new portfolio by copying the old one.
# We have to do it this way so the portfolio_id field on the setting is updated.
if hasattr(old_setting, 'portfolio'):
new_port = copy.copy(old_setting.portfolio)
pxs = new_port.items.all()
new_port.id = None
new_port.setting = setting
new_port.save()
for item in pxs:
item.id = None
item.portfolio = new_port
item.save()
else:
port_items_data = port_data.pop('items')
# Get the current portfolio statistics of the given weights.
try:
er, stdev, idatas = current_stats_from_weights(
[(item['asset'].id, item['weight'])
for item in port_items_data], DataProviderDjango())
except Unsatisfiable as e:
raise ValidationError(e.msg)
port = Portfolio.objects.create(setting=setting,
er=er,
stdev=stdev)
PortfolioItem.objects.bulk_create([
PortfolioItem(portfolio=port,
asset=i_data['asset'],
weight=i_data['weight'],
volatility=idatas[i_data['asset'].id])
for i_data in port_items_data
])
# Do the recurring transactions
if tx_data is None:
# We cannot simply reassign the transaction to the different setting, as the old setting may be used for
# the active or approved setting.
for item in old_setting.recurring_transactions.all():
new_tx = copy.copy(item)
new_tx.id = None
new_tx.setting = setting
new_tx.save()
else:
# validate i_data fields
for i_data in tx_data:
if 'enabled' not in i_data.keys():
raise ValidationError(
'Recurring transaction is missing enabled field.')
RecurringTransaction.objects.bulk_create([
RecurringTransaction(setting=setting,
schedule=i_data['schedule'],
enabled=i_data['enabled'],
begin_date=i_data['begin_date'],
growth=i_data['growth'],
amount=i_data['amount'],
account_id=i_data['account_id'])
for i_data in tx_data
])
try:
goal.set_selected(setting)
except CVE as verr:
raise ValidationError(verr.message)
return setting
def create_settings(plan):
"""
Creates some settings that can be used to create a real retirement goal if desired.
:param plan: The retirement plan to create settings for.
:return: A GoalSetting object that has been saved in the database.
The caller needs to destroy it if it is no longer required
:raises Unsatisfiable if no suitable portfolio could be found for the plan.
"""
metric_group = GoalMetricGroup.objects.create(type=GoalMetricGroup.TYPE_CUSTOM)
settings = GoalSetting.objects.create(
target=0,
completion=timezone.now().date(),
hedge_fx=False,
metric_group=metric_group,
)
risk_metric = GoalMetric.objects.create(group=metric_group,
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=plan.desired_risk)
# Create a mock goal so we can call calculate_portfolio
class MockGoal(object):
portfolio_set = plan.client.advisor.default_portfolio_set
id = 0
available_balance = 100000
current_balance = 100000
def __str__(self):
return "Retiresmartz calculation Goal for plan: {}".format(plan)
# Create a dummy settings object for the calculation.
# We need to do this because we are using a fake goal, as we don't have a real goal yet.
settings_data = GoalSettingSerializer(instance=settings).data
calc_settings = GoalSettingStatelessSerializer.create_stateless(settings_data, MockGoal())
data_provider = DataProviderDjango()
idata = get_instruments(data_provider)
weights, er, stdev = calculate_portfolio(
settings=calc_settings,
idata=idata,
data_provider=data_provider,
execution_provider=ExecutionProviderDjango()
)
portfolio = Portfolio.objects.create(
setting=settings,
stdev=stdev,
er=er,
)
items = [PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.items()]
PortfolioItem.objects.bulk_create(items)
return settings
def setUp(self):
Region.objects.create(name="AU")
Region.objects.create(name="UK")
Region.objects.create(name="US")
AssetClass.objects.create(name="US_BONDS",
investment_type=InvestmentType.Standard.BONDS.get(),
display_order=1)
AssetClass.objects.create(name="AU_STOCKS",
investment_type=InvestmentType.Standard.STOCKS.get(),
display_order=2)
AssetClass.objects.create(name="AU_STOCK_MUTUALS",
investment_type=InvestmentType.Standard.STOCKS.get(),
display_order=3)
index = MarketIndex.objects.create(id=1,
display_name='ASSIndex',
region=Region.objects.get(name="AU"),
currency='AUD',
data_api='bloomberg',
data_api_param='MI1')
ass = Ticker.objects.create(symbol="ASS",
display_name='AU Stock 1',
ethical=False,
region=Region.objects.get(name="AU"),
asset_class=AssetClass.objects.get(name='AU_STOCKS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='ASS')
ubs = Ticker.objects.create(symbol="USB",
display_name='US Bond 1',
ethical=True,
region=Region.objects.get(name="US"),
asset_class=AssetClass.objects.get(name='US_BONDS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='USB')
usb1 = Ticker.objects.create(symbol="USB1",
display_name='US Bond 2',
ethical=False,
region=Region.objects.get(name="US"),
asset_class=AssetClass.objects.get(name='US_BONDS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='USB1')
aums = Ticker.objects.create(symbol="AUMS",
display_name='AU Mutual Stocks 1',
ethical=True,
region=Region.objects.get(name="AU"),
asset_class=AssetClass.objects.get(name='AU_STOCK_MUTUALS'),
etf=False,
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='AUMS')
self._data_provider = DataProviderDjango()
self._execution_provider = ExecutionProviderDjango()
self._covars, self._samples, self._instruments, self._masks = build_instruments(self._data_provider)
MarkowitzScale.objects.create(date=self._data_provider.get_current_date(),
min=-1,
max=1,
a=1,
b=2,
c=3)
class BaseTest(TestCase):
def setUp(self):
Region.objects.create(name="AU")
Region.objects.create(name="UK")
Region.objects.create(name="US")
AssetClass.objects.create(name="US_BONDS",
investment_type=InvestmentType.Standard.BONDS.get(),
display_order=1)
AssetClass.objects.create(name="AU_STOCKS",
investment_type=InvestmentType.Standard.STOCKS.get(),
display_order=2)
AssetClass.objects.create(name="AU_STOCK_MUTUALS",
investment_type=InvestmentType.Standard.STOCKS.get(),
display_order=3)
index = MarketIndex.objects.create(id=1,
display_name='ASSIndex',
region=Region.objects.get(name="AU"),
currency='AUD',
data_api='bloomberg',
data_api_param='MI1')
ass = Ticker.objects.create(symbol="ASS",
display_name='AU Stock 1',
ethical=False,
region=Region.objects.get(name="AU"),
asset_class=AssetClass.objects.get(name='AU_STOCKS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='ASS')
ubs = Ticker.objects.create(symbol="USB",
display_name='US Bond 1',
ethical=True,
region=Region.objects.get(name="US"),
asset_class=AssetClass.objects.get(name='US_BONDS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='USB')
usb1 = Ticker.objects.create(symbol="USB1",
display_name='US Bond 2',
ethical=False,
region=Region.objects.get(name="US"),
asset_class=AssetClass.objects.get(name='US_BONDS'),
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='USB1')
aums = Ticker.objects.create(symbol="AUMS",
display_name='AU Mutual Stocks 1',
ethical=True,
region=Region.objects.get(name="AU"),
asset_class=AssetClass.objects.get(name='AU_STOCK_MUTUALS'),
etf=False,
ordering=1,
description='some stock',
benchmark=index,
data_api='portfolios.api.bloomberg',
data_api_param='AUMS')
self._data_provider = DataProviderDjango()
self._execution_provider = ExecutionProviderDjango()
self._covars, self._samples, self._instruments, self._masks = build_instruments(self._data_provider)
MarkowitzScale.objects.create(date=self._data_provider.get_current_date(),
min=-1,
max=1,
a=1,
b=2,
c=3)
def test_get_instruments(self):
#self.assertEqual(self._samples, 4)
self.assertEqual(int(self._instruments['mkt_cap'].values[0]), int(self._mkt_caps[-1]))
#self.assertListEqual(self._instruments['exp_ret'].values.tolist(), self._expected_returns)
def test_calculate_portfolio(self):
goal1 = Fixture1.goal1()
goal1.portfolio_set.asset_classes = [
AssetClass.objects.get(name="US_BONDS"),
AssetClass.objects.get(name="AU_STOCKS"),
AssetClass.objects.get(name="AU_STOCK_MUTUALS")
]
goal1.selected_settings.metric_group.metrics = [GoalMetric.objects.create(group=Fixture1.metric_group1(),
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
rebalance_type="1",
configured_val=0.0,
comparison=2,
rebalance_thr=0.05)
]
goal1.selected_settings.SYSTEM_CURRENCY = 'USD'
goal1.cash_balance = 1000
idata = get_instruments(self._data_provider)
portfolio, er, var = calculate_portfolio(settings=goal1.selected_settings,
data_provider=self._data_provider,
execution_provider=self._execution_provider,
idata=idata)
self.assertEqual(len(portfolio), 4)
def test_calculate_portfolios(self):
goal1 = Fixture1.goal1()
goal1.portfolio_set.asset_classes = [
AssetClass.objects.get(name="US_BONDS"),
AssetClass.objects.get(name="AU_STOCKS"),
AssetClass.objects.get(name="AU_STOCK_MUTUALS")
]
goal1.selected_settings.metric_group.metrics = [GoalMetric.objects.create(group=Fixture1.metric_group1(),
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
rebalance_type="1",
configured_val=0.0,
comparison=2,
rebalance_thr=0.05)
]
goal1.selected_settings.SYSTEM_CURRENCY = 'USD'
goal1.cash_balance = 1000
portfolio = Portfolio.objects.create(id=1, er=1, stdev=2, setting=goal1.selected_settings)
PortfolioItem.objects.create(asset=Ticker.objects.get(symbol='ASS'),
portfolio=portfolio,
weight=0.1,
volatility=0.2)
portfolios = calculate_portfolios(goal1.selected_settings, data_provider=self._data_provider,
execution_provider=self._execution_provider)
self.assertEqual(len(portfolios), 101)
def test_psd_norm(self):
data = {'AAXJ': [66.029999000000004, 63.0, 59.270000000000003, 53.340000000000003, 52.75],
'UBU': [20.079999999999998, 20.079999999999998, 21.550000000000001, 20.559999999999999, 20.18],
'ALD': [45.939999, 45.330002, 44.490001999999997, 42.729999999999997, 42.409999999999997],
'VSO': [47.399999999999999, 42.899999999999999, 43.340000000000003, 41.719999999999999, 40.950000000000003],
'VAS': [73.700000000000003, 69.989999999999995, 72.099999999999994, 66.569999999999993, 64.549999999999997],
'BTWJPNF_AU': [0.66000000000000003, 0.66000000000000003, 0.68999999999999995, 0.67000000000000004, 0.63],
'VGS': [59.75, 58.439999999999998, 61.0, 58.25, 56.780000000000001],
'EMB': [112.370003, 109.91999800000001, 109.660004, 108.010002, 106.400002],
'FTAL': [41.854999999999997, 39.329999999999998, 40.390000000000001, 38.32, 37.229999999999997],
'UBP': [20.150717539569801, 19.1999999999999, 19.050000000000001, 17.990000000000101, 17.240000000000101],
'BTWASHF_AU': [1.8799999999999999, 1.8400000000000001, 1.8799999999999999, 1.8400000000000001, 1.8400000000000001],
'VLC': [64.719999999999999, 61.219999999999999, 63.530000000000001, 57.469999999999999, 55.170000000000002],
'MCHI': [59.849997999999999, 56.040000999999997, 50.040000999999997, 44.099997999999999, 43.810001],
'UBE': [20.983828369806702, 20.140000000000001, 21.510000000000002, 20.1099999999999, 19.75],
'BTA0420_AU': [1.1799999999999999, 1.1299999999999999, 1.0700000000000001, 1.02, 1.0],
'SLXX': [136.13999999999999, 131.22, 134.57499999999999, 130.71000000000001, 131.46000000000001],
'VTS': [143.81, 139.49000000000001, 149.49000000000001, 143.16, 139.47],
'RGB': [21.379999999999999, 21.0, 21.280000000000001, 21.399999999999999, 21.52],
'IJP': [17.239999999999998, 16.710000000000001, 17.68, 16.98, 16.09],
'HOW0062_AU': [1.05, 1.05, 1.0, 1.01, 1.02],
'DSUM': [24.91, 24.739999999999998, 24.510000000000002, 23.040001, 23.559999000000001],
'ILB': [115.41, 113.8, 114.0, 114.56, 114.31999999999999],
'PEBIX_US': [9.9499999999999993, 10.529999999999999, 10.19, 10.1, 9.7400000000000002],
'BTWFAUS_AU': [1.74, 1.6499999999999999, 1.73, 1.5900000000000001, 1.5600000000000001],
'BTWEUSH_AU': [1.3200000000000001, 1.29, 1.3799999999999999, 1.3500000000000001, 1.3200000000000001],
'IEAG': [87.209999999999994, 83.355000000000004, 84.674999999999997, 87.055000000000007, 87.405000000000001],
'RSM': [20.789999999999999, 20.550000000000001, 20.77, 20.850000000000001, 20.629999999999999],
'ROTHWSE_AU': [2.6400000000000001, 2.4700000000000002, 2.3999999999999999, 2.3300000000000001, 2.3900000000000001],
'UBA': [19.886842423199901, 18.6400000000001, 19.129999999999999, 17.440000000000001, 16.879999999999999],
'IUSB': [101.769997, 100.519997, 100.459999, 100.389999, 100.25],
'ROTHFXD_AU': [1.23, 1.21, 1.1899999999999999, 1.21, 1.21],
'UBJ': [20.763995359855802, 20.479000000000099, 21.379999999999999, 20.549999999999901, 19.469999999999999],
'IEU': [61.130000000000003, 57.57, 61.130000000000003, 58.340000000000003, 56.100000000000001],
'VGE': [62.549999999999997, 60.229999999999997, 58.549999999999997, 53.600000000000001, 52.880000000000003],
'RIGS': [25.25, 24.940000999999999, 24.940000999999999, 24.549999, 24.100000000000001],
'VHY': [69.030000000000001, 65.040000000000006, 64.150000000000006, 59.219999999999999, 57.100000000000001],
'UBW': [21.244103679132198, 20.510000000000002, 21.620000000000001, 19.779999999999902, 20.079999999999998],
'BOND': [26.280000000000001, 25.800000000000001, 26.030000000000001, 26.23, 26.02],
'BTWAMSH_AU': [1.23, 1.21, 1.24, 1.21, 1.1799999999999999]}
df = pd.DataFrame(data)
df_cov = df.cov()
#print(df_cov)
p1 = cov_nearest(df_cov)
def test_calc_opt_inputs_no_assets_for_constraint(self):
"""
Makes sure when we have no assets filling a constraint, we behave appropriately.
"""
# This fund has a different feature to the one in the mix metric, but it is in the correct portfolio set.
fund1 = TickerFactory.create()
AssetFeatureValueFactory.create(assets=[fund1])
ps1 = PortfolioSetFactory.create(asset_classes=[fund1.asset_class])
# Create a settings object with a metric for a feature with no instruments in the current portfolio set.
feature = AssetFeatureValueFactory.create()
settings = GoalSettingFactory.create()
risk_metric = GoalMetricFactory.create(group=settings.metric_group)
mix_metric = GoalMetricFactory.create(
group=settings.metric_group,
type=GoalMetric.METRIC_TYPE_PORTFOLIO_MIX,
feature=feature,
comparison=GoalMetric.METRIC_COMPARISON_MAXIMUM,
configured_val=.3)
goal = GoalFactory.create(selected_settings=settings,
portfolio_set=ps1)
# The below fund has the desired feature, but is not in the goal's portfolio set.
fund2 = TickerFactory.create()
feature.assets.add(fund2)
# Create some instrument data for the two assets
self.m_scale = MarkowitzScaleFactory.create()
# populate the data needed for the prediction
# We need at least 500 days as the cycles go up to 70 days and we need at least 7 cycles.
populate_prices(500, asof=mocked_now.date())
populate_cycle_obs(500, asof=mocked_now.date())
populate_cycle_prediction(asof=mocked_now.date())
data_provider = DataProviderDjango()
idata = build_instruments(data_provider)
execution_provider = ExecutionProviderDjango()
# Get the opt inputs, there should be no constraint for the max for the feature with no funds.
result = calc_opt_inputs(settings=settings,
idata=idata,
data_provider=data_provider,
execution_provider=execution_provider)
xs, lam, constraints, settings_instruments, settings_symbol_ixs, lcovars = result
self.assertEqual(len(constraints), 3) # All positive, and sum to 1
# Then create a fund in the portfolio I want. We should get a constraint for the maximum for the feature.
fund3 = TickerFactory.create(asset_class=fund1.asset_class)
feature.assets.add(fund3)
delete_data()
populate_prices(500, asof=mocked_now.date())
populate_cycle_obs(500, asof=mocked_now.date())
populate_cycle_prediction(asof=mocked_now.date())
idata = build_instruments(data_provider)
result = calc_opt_inputs(settings=settings,
idata=idata,
data_provider=data_provider,
execution_provider=execution_provider)
xs, lam, constraints, settings_instruments, settings_symbol_ixs, lcovars = result
self.assertEqual(len(constraints),
4) # All positive, sum to 1, and the max constraint
def create(self, validated_data):
"""
Override the default create because we need to generate a portfolio.
:param validated_data:
:return: The created Goal
"""
account = validated_data['account']
data_provider = DataProviderDjango()
execution_provider = ExecutionProviderDjango()
idata = get_instruments(data_provider)
with transaction.atomic():
metric_group = GoalMetricGroup.objects.create(
type=GoalMetricGroup.TYPE_CUSTOM)
settings = GoalSetting.objects.create(
target=validated_data['target'],
completion=validated_data['completion'],
hedge_fx=False,
metric_group=metric_group,
)
portfolio_provider_id = validated_data[
'portfolio_provider'] if 'portfolio_provider' in validated_data else get_default_provider_id(
)
portfolio_set_id = ''
portfolio_providers = PortfolioProvider.objects.all()
portfolio_provider = get_default_provider()
for pp in portfolio_providers:
if pp.id == portfolio_provider_id:
portfolio_provider = pp
if portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_KRANE:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_KRANE
elif portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_AON:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_AON
elif portfolio_provider.type == constants.PORTFOLIO_PROVIDER_TYPE_LEE:
portfolio_set_type = constants.PORTFOLIO_SET_TYPE_LEE
else:
raise Exception('unhandled portfolio_provider_id')
portfolio_sets = PortfolioSet.objects.all()
portfolio_set = account.default_portfolio_set
for ps in portfolio_sets:
if ps.type == portfolio_set_type:
portfolio_set = ps
goal = Goal.objects.create(
account=account,
name=validated_data['name'],
type=validated_data['type'],
portfolio_set=portfolio_set,
portfolio_provider=portfolio_provider,
selected_settings=settings,
)
# Based on the risk profile, and whether an ethical profile was specified on creation, set up Metrics.
recommended_risk = recommend_risk(settings)
GoalMetric.objects.create(
group=metric_group,
type=GoalMetric.METRIC_TYPE_RISK_SCORE,
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=recommended_risk)
if validated_data['ethical']:
GoalMetric.objects.create(
group=metric_group,
type=GoalMetric.METRIC_TYPE_PORTFOLIO_MIX,
feature=AssetFeatureValue.Standard.SRI_OTHER.get_object(),
comparison=GoalMetric.METRIC_COMPARISON_EXACTLY,
rebalance_type=GoalMetric.REBALANCE_TYPE_ABSOLUTE,
rebalance_thr=0.05,
configured_val=1 # Start with 100% ethical.
)
# Make sure the risk score assigned is appropriate for the goal.
try:
validate_risk_score(settings)
except CVE as verr:
raise ValidationError(verr.message)
# Add the initial deposit if specified.
initial_dep = validated_data.pop('initial_deposit', None)
if initial_dep is not None:
Transaction.objects.create(reason=Transaction.REASON_DEPOSIT,
to_goal=goal,
amount=initial_dep)
# Calculate the optimised portfolio
try:
weights, er, stdev = calculate_portfolio(
settings, data_provider, execution_provider, True, idata)
portfolio = Portfolio.objects.create(
setting=settings,
stdev=stdev,
er=er,
)
items = [
PortfolioItem(portfolio=portfolio,
asset=Ticker.objects.get(id=tid),
weight=weight,
volatility=idata[0].loc[tid, tid])
for tid, weight in weights.iteritems()
]
PortfolioItem.objects.bulk_create(items)
except Unsatisfiable:
# We detect when loading a goal in the allocation screen if there has been no portfolio created
# and return a message to the user. It it perfectly reasonable for a goal to be created without a
# portfolio.
logger.exception(
"No suitable portfolio could be found. Leaving empty.")
return goal
