def originalDataSubSupport():
df = artificialDataGenerator.originalData()
print df.columns
df = df[[
'alcoholD', 'caffeineD', 'dairyP', 'eggP', 'fruitP', 'grainP', 'meatP',
'seafood', 'snack', 'starchyP', 'vegetables', 'entertainmentRelax',
'social', 'sport', 'transportation1', 'transportation2',
'transportation3', 'workStudy', 'gender', 'label', 'ID'
]]
df.columns = [
'alcohol', 'cafe', 'dairy', 'egg', 'fruit', 'grain', 'meat', 'seafood',
'snack', 'starchy', 'vegetables', 'leisure', 'social', 'sport', 'walk',
'car', 'bike', 'workStudy', 'gender', 'label', 'ID'
]
for i in range(df.shape[0]):
df.set_value(i, 'ID', int(df['ID'][i]))
sleep_list = [
'044', '045', '048', '050', '051', '052', '053', '056', '058', '059',
'060', '061', '063', '064', '065', '066', '067', '068', '069', '070',
'071', '072', '073', '074', '075'
]
dd_low_list = {}
dd_high_list = {}
dd_low = {}
dd_high = {}
for i in df.columns:
if i != 'label' and i != 'ID':
dd_low_list[i] = []
dd_high_list[i] = []
for sub in sleep_list:
ID = int(sub)
df_sub = df[df['ID'] == ID]
for i in df.columns:
if i != 'label' and i != 'ID':
temp = df_sub[df_sub[i] > 0]
if sum(df_sub[i]) != 0:
dd_low_list[i].append(
sum(temp[temp['label'] == 0][i]) / sum(df_sub[i]))
dd_high_list[i].append(
sum(temp[temp['label'] == 1][i]) / sum(df_sub[i]))
# else:
# dd_low_list[i].append(0)
# dd_high_list[i].append(0)
for i in dd_low_list.keys():
dd_low[i] = sum(dd_low_list[i]) / len(dd_low_list[i])
dd_high[i] = sum(dd_high_list[i]) / len(dd_high_list[i])
return dd_low, dd_high
def originalData():
df = artificialDataGenerator.originalData()
print df.columns
df = df[[
'alcoholD', 'caffeineD', 'dairyP', 'eggP', 'fruitP', 'grainP', 'meatP',
'seafood', 'snack', 'starchyP', 'vegetables', 'entertainmentRelax',
'social', 'sport', 'transportation1', 'transportation2',
'transportation3', 'workStudy', 'gender', 'label'
]]
df.columns = [
'alcohol', 'cafe', 'dairy', 'egg', 'fruit', 'grain', 'meat', 'seafood',
'snack', 'starchy', 'vegetables', 'leisure', 'social', 'sport', 'walk',
'car', 'bike', 'workStudy', 'gender', 'label'
]
dd = {}
dd_low = {}
dd_high = {}
dd_diff = {}
for i in df.columns:
if i != 'label':
dd[i] = [0, 0]
temp = df[df[i] > 0]
# dd[i][0] = temp[temp['label']==0].shape[0]
# dd[i][1] = temp[temp['label']==1].shape[0]
# dd[i][2] = temp[temp['label']==2].shape[0]
#
# dd[i][0] = sum(temp[temp['label']==0][i])
# dd[i][1] = sum(temp[temp['label']==1][i])
# dd[i][2] = sum(temp[temp['label']==2][i])
# dd[i][0] = sum(temp[temp['label']==0][i])/(temp[temp['label']==0].shape[0])
# dd[i][1] = sum(temp[temp['label']==1][i])/(temp[temp['label']==1].shape[0])
# dd[i][2] = sum(temp[temp['label']==2][i])/(temp[temp['label']==2].shape[0])
# ll = copy.deepcopy(dd)
# dd[i][0] = dd[i][0]/float(sum(ll[i]))
# dd[i][1] = dd[i][1]/float(sum(ll[i]))
# dd[i][2] = dd[i][2]/float(sum(ll[i]))
# dd_low[i] = dd[i][0] #+ dd[i][1]
# dd_high[i] = dd[i][2] #+ dd[i][1]
# dd_diff[i] = max(dd[i])-min(dd[i])
dd[i][0] = sum(temp[temp['label'] == 0][i]) / sum(df[i])
dd[i][1] = sum(temp[temp['label'] == 1][i]) / sum(df[i])
# dd[i][2] = sum(temp[temp['label']==2][i])/sum(df[i])
# dd[i][0] = float(temp[temp['label']==0][i].shape[0])/temp.shape[0]
# dd[i][1] = float(temp[temp['label']==1][i].shape[0])/temp.shape[0]
# dd[i][2] = float(temp[temp['label']==2][i].shape[0])/temp.shape[0]
dd_low[i] = dd[i][0]
dd_high[i] = dd[i][1]
dd_diff[i] = max(dd[i]) - min(dd[i])
return dd, dd_low, dd_high, dd_diff
def genOriginalActDietTypeDataSet():
dataset = []
df = artificialDataGenerator.originalData()
df.index = range(df.shape[0])
for i in range(df.shape[0]):
temp = []
for j in df.columns:
if j != 'compositeP' and j != 'label' and j!='sleepTime' and j!='gender' and j!='ID' and j!='others':
if df[j][i] > 0:
temp.append(j)
temp = tuple(temp)
dataset.append(temp)
dataset = tuple(dataset)
print len(dataset)
return dataset
def visdiff():
surrogateDF, labels = artificialDataGenerator.artificialData()
df, cols = artificialDataGenerator.originalData()
newDF = newFeatureFrame()
for i in newDF.columns:
plt.figure()
newDF[i].plot.kde(label='new')
if i == 'walk':
i = 'transportation1'
if i == 'car':
i = 'transportation2'
if i == 'bike':
i = 'transportation3'
if i == 'leisure':
i = 'entertainmentRelax'
df[i].plot.kde(label='original')
surrogateDF[i].plot.kde(label='surrogate')
# plt.legend()
plt.legend(bbox_to_anchor=(1.05, 1), loc=2)
plt.title(i)
plt.savefig('distribution/' + i + '_diff')
# accuracy = accuracy_score(labelTest,pre_labels)
#
# scores = cross_validation.cross_val_score(clf, dataset, labels, cv=5)
# accuracy = scores.mean()
#
# if accuracy > bestAcc:
# bestAcc = accuracy
# #p,r,f,s = precision_recall_fscore_support(labelTest,pre_labels)
#
# # p,r,f,s = precision_recall_fscore_support(labelTest,pre_labels)
# # cross_val_score(clf, dataset, labels)
# print bestAcc#,p,r,f,s
'''
original data test
'''
df = artificialDataGenerator.originalData()
temp_df = df[[
'compositeP', 'others', 'alcoholD', 'caffeineD', 'dairyP', 'eggP',
'fruitP', 'grainP', 'meatP', 'seafood', 'snack', 'starchyP', 'vegetables',
'entertainmentRelax', 'social', 'sport', 'transportation1',
'transportation2', 'transportation3', 'workStudy', 'gender'
]]
dataset = temp_df.as_matrix()
labels = list(df['label'])
clf = LogisticRegression(penalty='l1', C=0.5)
scores = cross_validation.cross_val_score(clf, dataset, labels, cv=5)
accuracy = scores.mean()
print accuracy
'''
original data test pattern features
'''
def clusteringKmeansLabelsOriginalDays():
workbookW = xlwt.Workbook()
ws = workbookW.add_sheet('sheet1')
rowW = 0
df,cols = artificialDataGenerator.originalData()
for domain in Domain:
print df.columns
if domain == 'DietType':
df_temp = df[['alcoholD','caffeineD','dairyP','eggP','fruitP','grainP','meatP','seafood','snack','starchyP','vegetables']]
row_labels = df_temp.columns
X = df_temp.as_matrix()
X = utilise.normArray(X)
kmeans = KMeans(n_clusters=2, n_init = 3000)
kmeans.fit(X)
labels = kmeans.labels_
else:
df_temp = df[['entertainmentRelax','social','sport','transportation1','transportation2','transportation3','workStudy']]
row_labels = df_temp.columns
X = df_temp.as_matrix()
X = utilise.normArray(X)
kmeans = KMeans(n_clusters=3, n_init = 3000)
kmeans.fit(X)
labels = kmeans.labels_
# write the lables to excel file
col = 0
for label in row_labels:
ws.write(rowW,col,label)
col += 1
rowW += 1
# print type(labels)
plt.figure()
n_clusters = np.max(labels) + 1
for k in range(n_clusters):
class_members = labels == k
group = []
for x in X[class_members]:
group.append(x)
group = np.array(group)
meanVec = np.mean(group,axis=0)
meanVec.tolist()
stdVec = np.std(group,axis=0)
stdVec.tolist()
print stdVec
# write the mean vector of each group to excel file
col = 0
for value in meanVec:
ws.write(rowW,col,value)
col += 1
rowW += 1
# print meanVec
firstMax = np.max(meanVec)
# print firstMax
tempVec = np.copy(meanVec)
for j in range(X.shape[1]):
if tempVec[j] == firstMax:
tempVec[j] = 0
secondMax = np.max(tempVec)
# print secondMax
tempVec2 = np.copy(tempVec)
for j in range(X.shape[1]):
if tempVec2[j]==secondMax:
tempVec2[j] = 0
thirdMax = np.max(tempVec2)
# print thirdMaxO
x = range(X.shape[1])
plt.plot(x,meanVec)
# plt.errorbar(x,meanVec,yerr=stdVec)
# print meanVec
for j in range(X.shape[1]):
# if meanVec[j] == firstMax:
# if meanVec[j] == firstMax or meanVec[j] == secondMax:
if meanVec[j] == firstMax or meanVec[j] == secondMax or meanVec[j] == thirdMax:
ws.write(rowW,0,k)
ws.write(rowW,1,domain)
ws.write(rowW,2,row_labels[j])
ws.write(rowW,3,meanVec[j])
rowW += 1
print k,domain,n_clusters,meanVec[j],row_labels[j]
if row_labels[j] == 'transportation1':
plt.text(x[j],meanVec[j],'walk')
elif row_labels[j] == 'transportation2':
plt.text(x[j],meanVec[j],'car')
elif row_labels[j] == 'transportation3':
plt.text(x[j],meanVec[j],'bike')
else:
plt.text(x[j],meanVec[j],row_labels[j])
# print row_labels
# plt.xlabel(row_labels)
plt.title(domain+'_TF_KMeans_'+str(n_clusters))
plt.savefig('visClustering'+domain+'Pattern/KMeans__TF_OriginalDays_'+str(n_clusters)+'_groupFreq')
workbookW.save('tempLabels.xls')