# This file optimizes our portfolio to make it sector neutral

import pandas as pd

import numpy as np

import ReadData

import functions

import datetime

import statsmodels.api as sm

import math

import matplotlib.pyplot as plt

from cvxopt import matrix, solvers, spmatrix

from matplotlib import pylab

data_folder = 'D:\Dropbox\CQA 2014\Data'

(StockPrices, SP500Prices, CarhartDaily, StockBeta) = ReadData.ReadAll(data_folder)

# StockPrices = StockPrices.drop(StockPrices.index[[92330]])

StockReturns = ReadData.ReadReturn(data_folder)

industry_data = pd.read_csv('%s\industry_data.csv' % data_folder)

first_date = StockPrices['date'].min()

last_date = StockPrices['date'].max()

stock_first = StockPrices[StockPrices['date'] == first_date][['ticker']].reset_index(drop=True)

stock_last = StockPrices[StockPrices['date'] == last_date][['ticker']].reset_index(drop=True)

stock_universe = stock_first.merge(stock_last).merge(StockBeta).merge(industry_data[['ticker', 'sector']])

tickers = stock_universe[['ticker']]

test_dates = np.sort(StockPrices['date'].astype(datetime.datetime).unique())

# build_start_date = datetime.datetime(2011, 11, 1)

# build_end_date = datetime.datetime(2014, 11, 6)

# test_start_date = datetime.datetime(2014, 11, 7)

build_start_date = test_dates[-1000]

build_end_date = test_dates[-250]

test_start_date = test_dates[-249]

test_end_date = last_date

decay = 0.003

prices_build = StockPrices[StockPrices['date'] == build_end_date][['ticker', 'adj_close']]

(Coef, Res) = functions.wReg(StockReturns, CarhartDaily, tickers, decay,

build_start_date, build_end_date)

betas = Coef.merge(tickers).rename(columns={'Mkt-RF': 'beta'})[['ticker', 'beta']].reset_index(drop=True)

sortCoef = betas.sort('beta').reset_index(drop=True)

N = sortCoef.shape[0] / 10

long_tickers = sortCoef.head(4*N).tail(3*N)[['ticker']].sort('ticker').reset_index(drop=True)

short_tickers = sortCoef.tail(4*N)[['ticker']].sort('ticker').reset_index(drop=True)

pfTickers = pd.concat((short_tickers[['ticker']], long_tickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True)

# def getCovMatrix(tickers):

CarhartSample = CarhartDaily[(CarhartDaily['date'] >= build_start_date) &

(CarhartDaily['date'] <= build_end_date)][['Mkt-RF', 'SMB', 'HML', 'UMD']].reset_index(drop=True)

ResSample = Res[pfTickers.ticker]

FactorCov = np.cov(CarhartSample.as_matrix().transpose())

# ResCov = np.cov(ResSample.as_matrix().transpose())

sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pfTickers).sort('ticker').drop('ticker', axis=1).as_matrix()

# totalCov = np.dot(np.dot(sens, FactorCov), sens.transpose()) + ResCov

totalCov = np.dot(np.dot(sens, FactorCov), sens.transpose())

BETA_BOUND = 0.3

WEIGHT_BOUND = 0.02

CAPITAL = 1e7

# SECTOR_WEIGHT = 0.05

# BAB, minimize 4-factor covariance matrix, maintain sector neutral

def pfOptimizer_sector(longTickers, shortTickers,

return wTickers

font_size = 20

color_cycle = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'lightblue', 'gold', 'pink']

k = 0

for SECTOR_WEIGHT in [0.1, 0.05, 0.04, 0.03, 0.02, 0.01, 0.005, 0.001]:

wTickers = pfOptimizer_sector(long_tickers, short_tickers,

Coef, StockBeta, industry_data,

BETA_BOUND, WEIGHT_BOUND, SECTOR_WEIGHT)

wTickers = wTickers[abs(wTickers['weight']) > 0.002]

portfolio = wTickers.merge(prices_build).rename(columns={'adj_close': 'price'})

portfolio['nShare'] = (portfolio['weight'] * CAPITAL / portfolio['price']).map(functions.intPart)

pfValues = functions.testPortfolio_simple(StockPrices, StockReturns, SP500Prices, StockBeta, portfolio, test_start_date)

pfValues['value'] = pfValues['value'] + CAPITAL - pfValues['value'][0]

plt.plot(pfValues['date'], pfValues['value'],

linewidth=2.0, color=color_cycle[k],

label='max sector weight %s' % SECTOR_WEIGHT)

k += 1

plt.title('Account Value vs Time', size=font_size)

plt.ylabel('Account Value', size=font_size)

plt.xlabel('Date', size=font_size)

plt.grid()

plt.legend(loc=2)

plt.show()