def getFrontier(rets, lRes=100, fUpper=0.2, fLower=0.00):
"""
@summary Generates an efficient frontier based on average returns.
@param rets: Array of returns to use
@param lRes: Resolution of the curve, default=100
@param fUpper: Upper bound on portfolio percentage
@param fLower: Lower bound on portfolio percentage
@return tuple containing (lf_ret, lfStd, lnaPortfolios)
lf_ret: List of returns provided by each point
lfStd: list of standard deviations provided by each point
lnaPortfolios: list of numpy arrays containing weights for each portfolio
"""
# Limit/enforce percent participation """
naUpper = np.ones(rets.shape[1]) * fUpper
naLower = np.ones(rets.shape[1]) * fLower
(fMin, fMax) = getRetRange(rets, naLower, naUpper)
# Try to avoid intractible endpoints due to rounding errors """
fMin *= 1.0000001
fMax *= 0.9999999
# Calculate target returns from min and max """
lf_ret = []
for i in range(lRes):
lf_ret.append((fMax - fMin) * i / (lRes - 1) + fMin)
lfStd = []
lnaPortfolios = []
# Call the function lRes times for the given range, use 1 for period """
for f_target in lf_ret:
(naWeights, fStd) = getOptPort(rets, f_target, 1,
naUpper=naUpper, naLower=naLower)
lfStd.append(fStd)
lnaPortfolios.append(naWeights)
# plot frontier """
#plt.plot( lfStd, lf_ret )
# there is no variable 'plt' in scope
#plt.plot(np.std(rets, axis=0), np.average(rets, axis=0),
# 'g+', markersize=10)
#plt.show()"""
return (lf_ret, lfStd, lnaPortfolios)
def returnize1(nds):
"""
@summary Computes stepwise (usually daily) returns relative to 1, where
1 implies no change in value.
@param nds: the array to fill backward
@return the array is revised in place
"""
s = np.shape(nds)
if len(s) == 1:
nds = np.expand_dims(nds, 1)
nds[1:, :] = (nds[1:, :] / nds[0:-1])
nds[0, :] = np.ones(nds.shape[1])
def stretchwfe(wfe,
factor):
Nx, Ny = wfe.shape
#if min(factor) lt 1 then message,'FACTOR needs to be greater than 1. Exiting.'
if factor.ndim != 2: print('FACTOR needs to be a 2d array. Exiting.')
Nx_new = np.round(Nx * factor[0])
Ny_new = np.round(Ny * factor[1])
wfe_new = zoom(wfe, factor, order = 1) # Order 1 = linear
# More pixels for same pattern
#Linear interpolation is important here because the discontinuity at
#the aperture edge would cause ringing if I used cubic interpolation.
#if wfe shrinks, pad with opaque aperture
if (Nx_new < Nx):
foo = 1e9j * np.ones([np.divide(Nx - Nx_new + 1, 2), Ny_new])
wfe_new = [foo, wfe_new, foo]
Nx_new = wfe_new.shape[0]
if Ny_new < Ny:
foo = 1e9j * np.ones([Nx_new, np.divide(Ny - Ny_new + 1, 2)])
wfe_new = [foo, wfe_new, foo]
Ny_new = wfe_new.shape[1]
#if wfe has expanded, cut it down. In fact, even if it shrunk, there may have
#been a slight expansion afterward due to off-by-half in the above padding!
x0 = 0.5 * (Nx_new - Nx)
x1 = x0 + Nx - 1
y0 = 0.5 * (Ny_new - Ny)
y1 = y0 + Ny - 1
return wfe_new[x0:x1, y0:y1]
def fit(self, X, y, verbose=False):
weights = np.ones(len(y)) / len(
y) # initial the weights for each data sample
accuracy = 0
epoch = 0
while 1 - accuracy > self.epsilon or epoch <= self.iters:
# stop when acc is okay or epoch number is larger than iters
epoch += 1
weak_learner = weak_learner_unit()
alpha = AdaBoost.calcAlpha(weak_learner.score(X, y, weights))
self.alphas.append(alpha)
self.weak_learners_list.append(weak_learner)
# update the weights by w :=
weights = weights * np.exp(-alpha * y * self.predict(X))
weights = weights / np.sum(weights)
accuracy = self.accuracy(X, y)
def fit(self, x, y, w=None):
if w is None:
w = np.ones(len(y)) / len(y)
data = zip(x, y, w)
data = sorted(data, key=lambda s: s[0])
[x, y, w] = zip(*data)
y = np.array(y)
w = np.array(w)
correct = np.zeros((2, len(y))) # 0 row for x < v, 1 row for x >= v
for i in range(len(y)):
w_front = w[:i]
w_back = w[i:]
correct[0, i] += np.sum(w_front[y[:i] == 1]) + np.sum(
w_back[y[i:] == -1])
correct[1, i] += np.sum(w_front[y[:i] == -1]) + np.sum(
w_back[y[i:] == 1])
idx = np.argmax(correct, axis=1)
if correct[0, int(idx[0])] > correct[1, int(idx[1])]:
self.sign = "smaller"
self.thres = x[idx[0]]
else:
self.sign = "equal to or bigger"
self.thres = x[idx[1]]
def __organize_output_data(self, data: np.array) -> np.ndarray:
return np.convolve(data[self.lag:], np.ones(self.forward), 'valid')
def optimizePortfolio(df_rets, list_min, list_max, list_price_target,
target_risk, direction="long"):
naLower = np.array(list_min)
naUpper = np.array(list_max)
naExpected = np.array(list_price_target)
b_same_flag = np.all(naExpected == naExpected[0])
if b_same_flag and (naExpected[0] == 0):
naExpected = naExpected + 0.1
if b_same_flag:
na_randomness = np.ones(naExpected.shape)
target_risk = 0
for i in range(len(na_randomness)):
if i % 2 == 0:
na_randomness[i] = -1
naExpected = naExpected + naExpected * 0.0000001 * na_randomness
(fMin, fMax) = getRetRange(df_rets.values, naLower, naUpper,
naExpected, direction)
# Try to avoid intractible endpoints due to rounding errors """
fMin += abs(fMin) * 0.00000000001
fMax -= abs(fMax) * 0.00000000001
if target_risk == 1:
(naPortWeights, fPortDev, b_error) = OptPort(df_rets.values, fMax, naLower, naUpper, naExpected, direction)
allocations = _create_dict(df_rets, naPortWeights)
return {'allocations': allocations, 'std_dev': fPortDev, 'expected_return': fMax, 'error': b_error}
fStep = (fMax - fMin) / 50.0
lfReturn = [fMin + x * fStep for x in range(51)]
lfStd = []
lnaPortfolios = []
for fTarget in lfReturn:
(naWeights, fStd, b_error) = OptPort(df_rets.values, fTarget, naLower, naUpper, naExpected, direction)
if not b_error:
lfStd.append(fStd)
lnaPortfolios.append(naWeights)
else:
# Return error on ANY failed optimization
allocations = _create_dict(df_rets, np.zeros(df_rets.shape[1]))
return {'allocations': allocations, 'std_dev': 0.0,
'expected_return': fMax, 'error': True}
if len(lfStd) == 0:
(naPortWeights, fPortDev, b_error) = OptPort(df_rets.values, fMax, naLower, naUpper, naExpected, direction)
allocations = _create_dict(df_rets, naPortWeights)
return {'allocations': allocations, 'std_dev': fPortDev, 'expected_return': fMax, 'error': True}
f_return = lfReturn[lfStd.index(min(lfStd))]
if target_risk == 0:
naPortWeights = lnaPortfolios[lfStd.index(min(lfStd))]
allocations = _create_dict(df_rets, naPortWeights)
return {'allocations': allocations, 'std_dev': min(lfStd), 'expected_return': f_return, 'error': False}
# Otherwise try to hit custom target between 0-1 min-max risk
fTarget = f_return + ((fMax - f_return) * target_risk)
(naPortWeights, fPortDev, b_error) = OptPort(df_rets.values, fTarget, naLower, naUpper, naExpected, direction)
allocations = _create_dict(df_rets, naPortWeights)
return {'allocations': allocations, 'std_dev': fPortDev, 'expected_return': fTarget, 'error': b_error}
def getRetRange(rets, naLower, naUpper, naExpected="False", s_type="long"):
"""
@summary Returns the range of possible returns with upper and lower bounds on the portfolio participation
@param rets: Expected returns
@param naLower: List of lower percentages by stock
@param naUpper: List of upper percentages by stock
@return tuple containing (fMin, fMax)
"""
# Calculate theoretical minimum and maximum theoretical returns """
fMin = 0
fMax = 0
rets = deepcopy(rets)
if naExpected == "False":
naExpected = np.average(rets, axis=0)
na_signs = np.sign(naExpected)
indices, = np.where(na_signs == 0)
na_signs[indices] = 1
if s_type == "long":
na_signs = np.ones(len(na_signs))
elif s_type == "short":
na_signs = np.ones(len(na_signs)) * (-1)
rets = na_signs * rets
naExpected = na_signs * naExpected
naSortInd = naExpected.argsort()
# First add the lower bounds on portfolio participation """
for i, fRet in enumerate(naExpected):
fMin = fMin + fRet * naLower[i]
fMax = fMax + fRet * naLower[i]
# Now calculate minimum returns"""
# allocate the max possible in worst performing equities """
# Subtract min since we have already counted it """
naUpperAdd = naUpper - naLower
fTotalPercent = np.sum(naLower[:])
for i, lInd in enumerate(naSortInd):
fRetAdd = naUpperAdd[lInd] * naExpected[lInd]
fTotalPercent = fTotalPercent + naUpperAdd[lInd]
fMin = fMin + fRetAdd
# Check if this additional percent puts us over the limit """
if fTotalPercent > 1.0:
fMin = fMin - naExpected[lInd] * (fTotalPercent - 1.0)
break
# Repeat for max, just reverse the sort, i.e. high to low """
naUpperAdd = naUpper - naLower
fTotalPercent = np.sum(naLower[:])
for i, lInd in enumerate(naSortInd[::-1]):
fRetAdd = naUpperAdd[lInd] * naExpected[lInd]
fTotalPercent = fTotalPercent + naUpperAdd[lInd]
fMax = fMax + fRetAdd
# Check if this additional percent puts us over the limit """
if fTotalPercent > 1.0:
fMax = fMax - naExpected[lInd] * (fTotalPercent - 1.0)
break
return (fMin, fMax)
def OptPort(naData, fTarget, naLower=None, naUpper=None, naExpected=None, s_type="long"):
"""
@summary Returns the Markowitz optimum portfolio for a specific return.
@param naData: Daily returns of the various stocks (using returnize1)
@param fTarget: Target return, i.e. 0.04 = 4% per period
@param lPeriod: Period to compress the returns to, e.g. 7 = weekly
@param naLower: List of floats which corresponds to lower portfolio% for each stock
@param naUpper: List of floats which corresponds to upper portfolio% for each stock
@return tuple: (weights of portfolio, min possible return, max possible return)
"""
''' Attempt to import library '''
try:
pass
from cvxopt import matrix
from cvxopt.solvers import qp, options
except ImportError:
print 'Could not import CVX library'
return ([], 0, True)
''' Get number of stocks '''
length = naData.shape[1]
b_error = False
naLower = deepcopy(naLower)
naUpper = deepcopy(naUpper)
naExpected = deepcopy(naExpected)
# Assuming AvgReturns as the expected returns if parameter is not specified
if (naExpected is None):
naExpected = np.average(naData, axis=0)
na_signs = np.sign(naExpected)
indices, = np.where(na_signs == 0)
na_signs[indices] = 1
if s_type == "long":
na_signs = np.ones(len(na_signs))
elif s_type == "short":
na_signs = np.ones(len(na_signs)) * (-1)
naData = na_signs * naData
naExpected = na_signs * naExpected
# Covariance matrix of the Data Set
naCov = np.cov(naData, rowvar=False)
# If length is one, just return 100% single symbol
if length == 0:
return ([], [0], False)
elif length == 1:
return (list(na_signs), np.std(naData, axis=0)[0], False)
# If we have 0/1 "free" equity we can't optimize
# We just use limits since we are stuck with 0 degrees of freedom
''' Special case for None == fTarget, simply return average returns and cov '''
if fTarget is None:
return (naExpected, np.std(naData, axis=0), b_error)
# Upper bound of the Weights of a equity, If not specified, assumed to be 1.
if naUpper is None:
naUpper = np.ones(length)
# Lower bound of the Weights of a equity, If not specified assumed to be 0 (No shorting case)
if naLower is None:
naLower = np.zeros(length)
if sum(naLower) == 1:
fPortDev = np.std(np.dot(naData, naLower))
return (naLower, fPortDev, False)
if sum(naUpper) == 1:
fPortDev = np.std(np.dot(naData, naUpper))
return (naUpper, fPortDev, False)
naFree = naUpper != naLower
if naFree.sum() <= 1:
lnaPortfolios = naUpper.copy()
# If there is 1 free we need to modify it to make the total
# Add up to 1
if naFree.sum() == 1:
f_rest = naUpper[~naFree].sum()
lnaPortfolios[naFree] = 1.0 - f_rest
lnaPortfolios = na_signs * lnaPortfolios
fPortDev = np.std(np.dot(naData, lnaPortfolios))
return (lnaPortfolios, fPortDev, False)
# Double the covariance of the diagonal elements for calculating risk.
for i in range(length):
naCov[i][i] = 2 * naCov[i][i]
# Note, returns are modified to all be long from here on out
(fMin, fMax) = getRetRange(False, naLower, naUpper, naExpected, "long")
#print (fTarget, fMin, fMax)
if fTarget < fMin or fTarget > fMax:
print "Target not possible", fTarget, fMin, fMax
b_error = True
naLower = naLower * (-1)
# Setting up the parameters for the CVXOPT Library, it takes inputs in Matrix format.
'''
The Risk minimization problem is a standard Quadratic Programming problem according to the Markowitz Theory.
'''
S = matrix(naCov)
#pbar=matrix(naExpected)
naLower.shape = (length, 1)
naUpper.shape = (length, 1)
naExpected.shape = (1, length)
zeo = matrix(0.0, (length, 1))
I = np.eye(length)
minusI = -1 * I
G = matrix(np.vstack((I, minusI)))
h = matrix(np.vstack((naUpper, naLower)))
ones = matrix(1.0, (1, length))
A = matrix(np.vstack((naExpected, ones)))
b = matrix([float(fTarget), 1.0])
# Optional Settings for CVXOPT
options['show_progress'] = False
options['abstol'] = 1e-25
options['reltol'] = 1e-24
options['feastol'] = 1e-25
# Optimization Calls
# Optimal Portfolio
try:
lnaPortfolios = qp(S, -zeo, G, h, A, b)['x']
except:
b_error = True
if b_error:
print "Optimization not Possible"
na_port = naLower * -1
if sum(na_port) < 1:
if sum(naUpper) == 1:
na_port = naUpper
else:
i = 0
while(sum(na_port) < 1 and i < 25):
naOrder = naUpper - na_port
i = i + 1
indices = np.where(naOrder > 0)
na_port[indices] = na_port[indices] + (1 - sum(na_port)) / len(indices[0])
naOrder = naUpper - na_port
indices = np.where(naOrder < 0)
na_port[indices] = naUpper[indices]
lnaPortfolios = matrix(na_port)
lnaPortfolios = (na_signs.reshape(-1, 1) * lnaPortfolios).reshape(-1)
# Expected Return of the Portfolio
# lfReturn = dot(pbar, lnaPortfolios)
# Risk of the portfolio
fPortDev = np.std(np.dot(naData, lnaPortfolios))
return (lnaPortfolios, fPortDev, b_error)
def getOptPort(rets, f_target, l_period=1, naLower=None, naUpper=None, lNagDebug=0):
"""
@summary Returns the Markowitz optimum portfolio for a specific return.
@param rets: Daily returns of the various stocks (using returnize1)
@param f_target: Target return, i.e. 0.04 = 4% per period
@param l_period: Period to compress the returns to, e.g. 7 = weekly
@param naLower: List of floats which corresponds to lower portfolio% for each stock
@param naUpper: List of floats which corresponds to upper portfolio% for each stock
@return tuple: (weights of portfolio, min possible return, max possible return)
"""
# Attempt to import library """
try:
pass
import nagint as nag
except ImportError:
print 'Could not import NAG library'
print 'make sure nagint.so is in your python path'
return ([], 0, 0)
# Get number of stocks """
lStocks = rets.shape[1]
# If period != 1 we need to restructure the data """
if(l_period != 1):
rets = getReindexedRets(rets, l_period)
# Calculate means and covariance """
naAvgRets = np.average(rets, axis=0)
naCov = np.cov(rets, rowvar=False)
# Special case for None == f_target"""
# simply return average returns and cov """
if(f_target is None):
return naAvgRets, np.std(rets, axis=0)
# Calculate upper and lower limits of variables as well as constraints """
if(naUpper is None):
naUpper = np.ones(lStocks) # max portfolio % is 1
if(naLower is None):
naLower = np.zeros(lStocks) # min is 0, set negative for shorting
# Two extra constraints for linear conditions"""
# result = desired return, and sum of weights = 1 """
naUpper = np.append(naUpper, [f_target, 1.0])
naLower = np.append(naLower, [f_target, 1.0])
# Initial estimate of portfolio """
naInitial = np.array([1.0 / lStocks] * lStocks)
# Set up constraints matrix"""
# composed of expected returns in row one, unity row in row two """
naConstraints = np.vstack((naAvgRets, np.ones(lStocks)))
# Get portfolio weights, last entry in array is actually variance """
try:
naReturn = nag.optPort(naConstraints, naLower, naUpper,
naCov, naInitial, lNagDebug)
except RuntimeError:
print 'NAG Runtime error with target: %.02lf' % (f_target)
return (naInitial, sqrt(naCov[0][0]))
#return semi-junk to not mess up the rest of the plot
# Calculate stdev of entire portfolio to return"""
# what NAG returns is slightly different """
fPortDev = np.std(np.dot(rets, naReturn[0, 0:-1]))
# Show difference between above stdev and sqrt NAG covariance"""
# possibly not taking correlation into account """
#print fPortDev / sqrt(naReturn[0, -1])
# Return weights and stdDev of portfolio."""
# note again the last value of naReturn is NAG's reported variance """
return (naReturn[0, 0:-1], fPortDev)
def score(self, x, y, w=None): # the wrong percent
if w is None:
w = np.ones(len(y)) / len(y)
return 1 - np.sum(w[self.predict(x) == y])