def standardize(df):
X = df.drop(['id', 'group'], axis=1)
X.ix[X.gender == 'f', 'gender'] = 1.0
X.ix[X.gender == 'm', 'gender'] = -1.0
X['gender'] = pd.to_numeric(X.gender)
X = StandardScaler().fit_transform(X.as_matrix())
return X
# Encode the nta column
le = LabelEncoder()
nta_encoded = le.fit_transform(data.nta)
data['nta_encoded'] = nta_encoded
# Drop the columns we won't use for our model
data.drop(['nta','date_hour','nta_dt','nbhd_name','Unnamed: 0'],axis=1,inplace=True)
# Scale our data (not every column needs to be scaled)
to_scale = data[(data.columns-['month','day','hour','nta_encoded'])]
X = StandardScaler().fit_transform(to_scale)
X = pd.DataFrame(X)
for col in ['month','day','hour','nta_encoded']:
X[col] = data[col]
X = X.as_matrix()
# We want to determine the number of bins we are going to classify into by using
# k-means clustering different values of n clusters. We are going to choose the
# n with the best silhouette score
cluster_scores = []
for n in range(3,6):
# Initialize and fit a KMeans object
k_means = cluster.KMeans(n_clusters=n,n_jobs=-1,verbose=1)
k_means.fit(X)
# Get the cluster labels for each point
labels = k_means.labels_
# Initialize a list to store this n's silhouette scores
scores = []
# We need to limit the sample_size when calculating silhouette scores due to
# computation time