# -*- coding: utf-8 -*-

"""

Created on Fri Jun 03 12:27:51 2016

@author: Peter

"""

import os

import pandas

import numpy

import sklearn

import matplotlib

import matplotlib.pyplot as plt

import sys; print(sys.path)

from seaborn import *

import seaborn as sns

import ggplot

from ggplot import *

import scipy

from pandas import Series, DataFrame

import pandas as pd

import numpy as np

import os

import matplotlib.pylab as plt

from sklearn.cross_validation import train_test_split

from sklearn.tree import DecisionTreeClassifier

from sklearn.metrics import classification_report

import sklearn.metrics

# Feature Importance

from sklearn.ensemble import ExtraTreesClassifier

import pydot

import graphviz

apath='C:\Users\Peter\Desktop\Gapminder'

print(apath)

os.chdir('C:\Users\Peter\Desktop\Gapminder')

##check the directory has changed

os.getcwd()

##read in the file

data = pandas.read_csv('gapminder.csv', low_memory=False)

##lets convert the data to numeric

data['incomeperperson'] = data['incomeperperson'].convert_objects(convert_numeric=True)

data['alcconsumption'] = data['alcconsumption'].convert_objects(convert_numeric=True)

data['armedforcesrate'] = data['armedforcesrate'].convert_objects(convert_numeric=True)

data['breastcancerper100th'] = data['breastcancerper100th'].convert_objects(convert_numeric=True)

data['co2emissions'] = data['co2emissions'].convert_objects(convert_numeric=True)

data['femaleemployrate'] = data['femaleemployrate'].convert_objects(convert_numeric=True)

data['hivrate'] = data['hivrate'].convert_objects(convert_numeric=True)

data['internetuserate'] = data['internetuserate'].convert_objects(convert_numeric=True)

data['lifeexpectancy'] = data['lifeexpectancy'].convert_objects(convert_numeric=True)

data['oilperperson'] = data['oilperperson'].convert_objects(convert_numeric=True)

data['polityscore'] = data['polityscore'].convert_objects(convert_numeric=True)

data['relectricperperson'] = data['relectricperperson'].convert_objects(convert_numeric=True)

data['suicideper100th'] = data['suicideper100th'].convert_objects(convert_numeric=True)

data['employrate'] = data['employrate'].convert_objects(convert_numeric=True)

data['urbanrate'] = data['urbanrate'].convert_objects(convert_numeric=True)

bins = [0, 1000, 5000, 10000, 20000,50000,200000]

group_names = ['Very Low Income,0-1000', 'Low Income,1000-5000', 'Okay Income,5000-10000', 'Good Income,10000-20000','Great Income,20000-50000','50,000-200,000']

categories = pandas.cut(data['incomeperperson'], bins, labels=group_names)

data['categories'] = pandas.cut(data['incomeperperson'], bins, labels=group_names)

##data.dtypes chk

##now encode european countries

def EUROPEAN (row):

return 'Not_In_Europe'

data['European'] = data.apply (lambda row: EUROPEAN (row),axis=1)

##check it worked

##'''

##Out[24]:

##Europe 45

##Not-In-Europe 168

##dtype: int64

##'''

data['European'].value_counts(sort=False, dropna=False)

data['country']

##checked working

def African (row):

return 'Not_In_Africa'

data['African'] = data.apply (lambda row: African (row),axis=1)

data['African'].value_counts(sort=False, dropna=False)

def Asian (row):

return 'Not_In_Asia'

data['Asian'] = data.apply (lambda row: Asian (row),axis=1)

data['Asian'].value_counts(sort=False, dropna=False)

##data[['country','incomeperperson','polityscore_cat']][(data.European=='Europe') & (data.polityscore_cat!='NA') ]

##data[(data.Asian=='Asian')]

def Mid_East (row):

return 'Not_In_Middle_East'

data['Mid_East'] = data.apply (lambda row: Mid_East (row),axis=1)

data['Mid_East'].value_counts(sort=False, dropna=False)

def North_American (row):

return 'Not_In_North_America'

data['North_American'] = data.apply (lambda row: North_American (row),axis=1)

data['North_American'].value_counts(sort=False, dropna=False)

def Carribean_Central_America (row):

return 'Not_In_Carribean_Central_American'

data['Carribean_Central_America'] = data.apply (lambda row: Carribean_Central_America (row),axis=1)

data['Carribean_Central_America'].value_counts(sort=False, dropna=False)

##Algeria, Angola, Ecuador, Iran, Iraq, Kuwait, Libya, Nigeria, Qatar, Saudi Arabia, United Arab Emirates and Venezuela

def OPEC (row):

return 'Not_In_OPEC'

data['OPEC'] = data.apply (lambda row: OPEC (row),axis=1)

data['OPEC'].value_counts(sort=False, dropna=False)

def Arab_League (row):

return 'Not_In_Arab_League'

##

data['Arab_League'] = data.apply (lambda row: Arab_League (row),axis=1)

data['Arab_League'].value_counts(sort=False, dropna=False)

##ASEAN is a regional grouping with security, economic and social aspects

def ASEAN_ARF (row):

return 'Not_In_ASEAN_ARF'

data['ASEAN_ARF'] = data.apply (lambda row: ASEAN_ARF (row),axis=1)

data['ASEAN_ARF'].value_counts(sort=False, dropna=False)

def South_American (row):

return 'Not_South_America'

data['South_American'] = data.apply (lambda row: South_American (row),axis=1)

data['South_American'].value_counts(sort=False, dropna=False)

##

###http://www.nato.int/cps/en/natohq/topics_52044.htm

##NATO data

Nato_Countries = pandas.DataFrame({ 'country' : ('Albania','Belgium','Bulgaria','Canada','Croatia','Czech Republic','Denmark','Estonia','France','Germany','Greece','Hungary','Iceland','Italy','Latvia','Lithuania','Luxembourg','Netherlands','Norway','Poland','Portugal','Romania','Slovak Republic','Slovenia','Spain','Turkey','United Kingdom','United States'),

'Year_Joined' : (2009,1949,2004,1949,2009,1999,1949,2004,1949,1955,1952,1999,1949,1949,2004,2004,1949,1949,1949,1999,1949,2004,2004,2004,1982,1952,1949,1949),

'Is_Nato_Country' : 'Nato_Member'

})

##Enhanced data join NATO data

data.columns.values

data=pandas.merge(data, Nato_Countries,how='left',on='country')

##data.columns.values

##check that all column values have been added

data['Is_Nato_Country']=data['Is_Nato_Country'].fillna('Not_in_Nato')

##

data['Is_Nato_Country'].value_counts(sort=False, dropna=False)

data.columns.values

##year joined needs to be renamed

data.rename(columns={'Year_Joined': 'Year_Joined_Nato'}, inplace=True)

## change columns names

data.columns.values

def EUMEMBER (row):

return 'Not_In_EU'

data['Eu_Member'] = data.apply (lambda row: EUMEMBER (row),axis=1)

data['Eu_Member'].value_counts(sort=False, dropna=False)

data.columns.values

import time

##check how to calc time

print (time.strftime("%Y"))

##write unction to calculate the age of NATO countries based on the current date

def AGE_YEARS (row):

return 0

##calculate the age of NATO countries

data['Years_In_Nato'] = data.apply (lambda row: AGE_YEARS (row),axis=1)

##calculate the age of NAto countries

print("distribution of years in NATO for Nato Countries")

pp2=data['Years_In_Nato'].value_counts(sort=False, dropna=False)

print(pp2)

##Mean number of years in NATO by european or Non EU

print("Mean years of countries in NATO for European and Non European Countries")

pp3=data[data['Years_In_Nato']>0]['Years_In_Nato'].groupby(data['Eu_Member']).mean()

print(pp3)

##Count of countries in EU who are not in not in NATO

print("Count of countries in NATO for European and Non European Countries")

pp4=data[data['Years_In_Nato']>0]['Years_In_Nato'].groupby(data['Eu_Member']).count()

print(pp4)

def EU_NATO (Nato_Membership,EU_Membership):

return 'Not_In_Nato_Not_In_EU'

EU_NATO('Nato_Member','EU')

##test

##apply the function

data['NATO_EU_MEMBERSHIP'] = data.apply (lambda row: EU_NATO(row['Is_Nato_Country'],row['Eu_Member']),axis=1)

data.columns.values

def polityscore_cat (row):

return 'NA'

##calculate the age of NATO countries

##data['Years_In_Nato'] = data.apply (lambda row: AGE_YEARS (row),axis=1)

data['polityscore_cat'] = data.apply (lambda row: polityscore_cat (row),axis=1)

data.columns.values

##array(['country', 'incomeperperson', 'alcconsumption', 'armedforcesrate',

## 'breastcancerper100th', 'co2emissions', 'femaleemployrate',

## 'hivrate', 'internetuserate', 'lifeexpectancy', 'oilperperson',

## 'polityscore', 'relectricperperson', 'suicideper100th',

## 'employrate', 'urbanrate', 'categories', 'European', 'African',

## 'Asian', 'Mid_East', 'North_American', 'Carribean_Central_America',

## 'OPEC', 'Arab_League', 'ASEAN_ARF', 'South_American',

## 'Is_Nato_Country', 'Year_Joined_Nato', 'Eu_Member', 'Years_In_Nato',

## 'NATO_EU_MEMBERSHIP', 'polityscore_cat'], dtype=object)

##data = data.drop('Is_Nato_Country_y', 1)

##data.rename(columns={'Is_Nato_Country_x': 'Is_Nato_Country'}, inplace=True)

data['European'].value_counts(sort=False, dropna=False)

data['European'].replace("Europe",1,inplace=True)

data['European'].replace("Not_In_Europe",0,inplace=True)

data['African'].value_counts(sort=False, dropna=False)

data['African'].replace("Africa",1,inplace=True)

data['African'].replace("Not_In_Africa",0,inplace=True)

data['African'].value_counts(sort=False, dropna=False)

data['Asian'].value_counts(sort=False, dropna=False)

data['Asian'].replace("Asia",1,inplace=True)

data['Asian'].replace("Not_In_Asia",0,inplace=True)

data['Asian'].value_counts(sort=False, dropna=False)

##'Mid_East'

data['Mid_East'].value_counts(sort=False, dropna=False)

data['Mid_East'].replace("Middle_East",1,inplace=True)

data['Mid_East'].replace("Not_In_Middle_East",0,inplace=True)

data['Mid_East'].value_counts(sort=False, dropna=False)

data['North_American'].value_counts(sort=False, dropna=False)

data['North_American'].replace("North_America",1,inplace=True)

data['North_American'].replace("Not_In_North_America",0,inplace=True)

data['North_American'].value_counts(sort=False, dropna=False)

data['Carribean_Central_America'].value_counts(sort=False, dropna=False)

data['Carribean_Central_America'].replace("Carribean_Central_American",1,inplace=True)

data['Carribean_Central_America'].replace("Not_In_Carribean_Central_American",0,inplace=True)

data['Carribean_Central_America'].value_counts(sort=False, dropna=False)

data['OPEC'].replace("OPEC_MEMBER",1,inplace=True)

data['OPEC'].replace("Not_In_OPEC",0,inplace=True)

data['OPEC'].value_counts(sort=False, dropna=False)

data['Arab_League'].value_counts(sort=False, dropna=False)

data['Arab_League'].replace("Not_In_Arab_League",0,inplace=True)

data['Arab_League'].replace("Arab_League_MEMBER",1,inplace=True)

data['Arab_League'].value_counts(sort=False, dropna=False)

##'ASEAN_ARF'

data['ASEAN_ARF'].value_counts(sort=False, dropna=False)

data['ASEAN_ARF'].replace("Not_In_ASEAN_ARF",0,inplace=True)

data['ASEAN_ARF'].replace("ASEAN_ARF_MEMBER",1,inplace=True)