#! /usr/bin/env python

## trumptrends.py

import sys

import pickle

import traceback

import time

import random

import string

import os.path

import glob

import textwrap

import numpy as np

import pandas as pd

from pandas.plotting import autocorrelation_plot

import matplotlib.pyplot as plt

from matplotlib.offsetbox import AnchoredOffsetbox, DrawingArea, TextArea, HPacker, VPacker, AnnotationBbox

from matplotlib.text import Text

from matplotlib.patches import Rectangle

import matplotlib.transforms as transforms

from matplotlib.dates import MinuteLocator, HourLocator, DayLocator, WeekdayLocator, MonthLocator, YearLocator, AutoDateLocator

from matplotlib.dates import AutoDateFormatter, DateFormatter, IndexDateFormatter

from matplotlib.ticker import NullLocator, FixedLocator, NullFormatter

import statsmodels.formula.api as sm

from scipy.ndimage.filters import gaussian_filter1d as gaussfilt

from sklearn.decomposition import FastICA, PCA

from sklearn.preprocessing import MinMaxScaler

from scipy.stats import skew

# Key dates:

keydates = {pd.to_datetime('2016-06-09'):'Obama Endorses Hillary',

pd.to_datetime('2017-05-31'):'Trump Tweets "Covfefe"'}

# 2017 March 2: Jeff Sessions recuses himself from Russia investigations

# 2017 March 4: Trump accuses Obama of wiretapping him

# 2017 March 21: bunch of things http://www.cbsnews.com/news/latest-donald-trump-news-today-march-21-2017/

# 2017 March 31: bunch of things http://www.cbsnews.com/news/latest-donald-trump-news-today-march-31-2017/

# less important 2016-06-18

keydatesh= {pd.to_datetime('2017-05-31'):'Trump Tweets "Covfefe"',

pd.to_datetime('2017-06-03'):'March for Truth'}

###############################################

## Utility functions to use with components ##

###############################################

def peakDirection(aseries, median=False):

return pdir

def peakiness(aseries, median=False):

return pness

###############################################

## Utility functions to use for plotting ##

###############################################

def format_termlist(termlist, wrap=40, ncol=2):

return outlist

def bestworstwords(component, useabs=False, nterms=20, wrap=20):

return (cw1, cw2, component)

###############################################

## The big components plotting function ##

###############################################

def componentplot(fittrans, components, figtitle, show=False, filename='icacomponents.pdf', keydates=None, nterms=20, figsize=(8,10), dpi=200, fontsize=None, rate='daily'):

fig.savefig(filename)

##################################################

## Now let's have some fun with those functions ##

##################################################

# Load all the individual df results in the data dir

#datadir = 'data01'

#datadir = 'hourlydata01'

# filepattern = os.path.join(datadir, '*.pkl.gz')

# filelist = glob.glob(filepattern)

# dflistAll = []

# dfdict = {}

# for f in filelist:

# searchterm = f[:f.index('.')]

# df = pd.read_pickle(f)

# dflistAll.append(df)

# dfdict[searchterm]=df

# # now make one master df

# bigdf = pd.concat(dflistAll, axis=1)

# # the list of dates, for re-indexing future data frames

# # the list of terms, for re-indexing future data frames

# minmaxscaler = MinMaxScaler(feature_range=(0,100))

# bigdf = minmaxscaler.fit_transform(bigdf)

# bigdf = pd.DataFrame(bigdf)

# bigdf.index = dfDateIndex

# bigdf.columns = dfSearchTermIndex

#

# bigdf.columns = [c.replace(' ','_') for c in bigdf.columns]

# #print(bigdf.columns)

###############################################

## Number of components to use in ICA ##

###############################################

ncomp = 10

###############################################

def myICA(bigdf, ncomp, whiten=True):

return icafittrans, icacomponents

def myPCA(bigdf, ncomp, whiten=True):

pcacomponents = pcacomponents.iloc[peakinessSort,:]

#datafile = 'hourlydata01.csv'

datafile = 'dailydata02.csv'

bigdf = pd.read_csv(datafile, index_col=0, parse_dates=True)

print('Master df bigdf has shape: (%d, %d)' % bigdf.shape)

totalterms = bigdf.shape[1]

print('Number of search terms: %d' % totalterms)

dfDateIndex=bigdf.index

dfSearchTermIndex = bigdf.columns

# number of terms to show for each component, for reference

nterms = 28

#figsize = [8,10]

figsize = [12,15]

dpi = 200 # don't really need this for pdf

figtitle = 'Independent Components Analysis of Search Trends for %d Political Terms' % bigdf.shape[1]

# With ICA, the fit of the model depends on the assumed number of components.

# The components generated by the model are different if a different number is requested.

#complist = [8, 9, 10, 11, 12, 13, 14, 15, 16]

#complist=[8]

complist=[13]

#complist=[6]

for i, ncomp in enumerate(complist):

icafittrans, icacomponents = myICA(bigdf, ncomp)

filename = 'components plots/icacomponents%0d.pdf' % ncomp

#print([i.strftime('%b %d') for i in icafittrans.index])

componentplot(icafittrans, icacomponents, figtitle, show=False, filename=filename, keydates=keydates, nterms=nterms, figsize=figsize, dpi=dpi, fontsize=None, rate='hourly')

sys.exit()

figtitle = 'Principal Components Analysis of Search Trends for %d Political Terms' % bigdf.shape[1]

# With PCA, adding more components doesn't change the existing ones, so there's no need to do multiple runs.

complist=[16]

for i, ncomp in enumerate(complist):

pcafittrans, pcacomponents = myPCA(bigdf, ncomp)

filename = 'components plots/pcacomponents%0d.pdf' % ncomp

componentplot(pcafittrans, pcacomponents, figtitle, show=False, filename=filename, keydates=keydates, nterms=nterms, figsize=figsize, dpi=dpi, fontsize=None)

sys.exit()

# Let's do a PCA on all this data, see what it looks like

pca = PCA(n_components=ncomp, whiten=True)

pcafit = pca.fit(bigdf.iloc[:,:])

pcafittrans = pca.transform(bigdf)

pcafittrans = pd.DataFrame(pcafittrans)

pcafittrans.index = dfDateIndex

print(pcafittrans.shape)

fig=plt.figure()

ax=fig.add_subplot(111)

ax.plot(pcafit.explained_variance_ratio_)

ax.set_title('PCA Scree Plot for full data')

plt.show()

# for pca, the "mixing" matrix should be this:

sys.exit()

sig = 4

meansdfSmooth = meansdf.copy()

print('Smoothing means into new data frame')

for column in meansdfSmooth:

meansdfSmooth[column] = gaussfilt(meansdf[column], sigma=sig)

plt.plot(meansdf['DTHealth'],meansdf['DTDementia'])

plt.show()

plt.plot(meansdfSmooth['DTHealth'],meansdfSmooth['DTDementia'])

plt.show()

meansdfSmooth['DTDemMinusHel'] = meansdfSmooth['DTDementia'] - meansdfSmooth['DTHealth']

meansdfSmooth['DTDemMinusHel'].plot()

plt.show()

ax=meansdf[['DTHealth','DTDementia']].plot()

meansdfSmooth[['DTHealth','DTDementia']].plot(ax=ax)

plt.show()

#meansdfSmooth = pd.DataFrame({'DTDementia':gaussfilt(dtdemMean, sigma=sig), 'HCDementia':gaussfilt(hcdemMean, sigma=sig), 'DTHealth':gaussfilt(dthelMean, sigma=sig), 'HCHealth':gaussfilt(hchelMean, sigma=sig), 'BOHealth':gaussfilt(bohelMean, sigma=sig)})

meansdfSmooth.plot()

#plt.show()

#print(meansdf)

sys.exit()