def setDict_Indices(self):
'''Once raw cluster/label dicts are created, assign indices to them'''
# Assign indices to class: Order of "inverse length", then by "key ascending"
for clusterRaw, dataSamples in H.sortDictLen_Rev(self.dict_ClusterRaw_DataSamples):
self.dict_Cluster_Index[clusterRaw] = len(self.dict_Cluster_Index) + 1
# Assign indices to labels: Order of "inverse length", then by "key ascending"
for label, dataSamples in H.sortDictLen_Rev(self.dict_Label_DataSamples):
self.dict_Label_Index[label] = len(self.dict_Label_Index) + 1
self.dict_Index_Cluster = {v: k for k, v in self.dict_Cluster_Index.iteritems()}
self.dict_Index_Label = {v: k for k, v in self.dict_Label_Index.iteritems()}
def splitTrainTest(self):
print colored('\tSplitting Train+Test ...', 'magenta')
self.num_classes = len(self.dict_ClusterRaw_DataSamples) + 1
self.num_labels = len(self.dict_Label_DataSamples)
print 'NumClasses=', self.num_classes - 1
print 'NumLabels=', self.num_labels - 1
for clusterRaw, dataSamples in H.sortDictLen_Rev(self.dict_ClusterRaw_DataSamples):
clusterIndex = self.dict_Cluster_Index[clusterRaw]
li = dataSamples
labelList = li[0].labelList # Pick any dataSamples labelList - would be the same for all similar clusterRaw
labelIndices = [self.dict_Label_Index[label] for label in labelList]
numTrain = int(math.ceil(CF.TRAIN_SPLIT * len(li)))
numValid = int(math.floor(CF.VALIDATION_SPLIT * len(li)))
numTest = len(li) - numTrain - numValid
print 'Class-',clusterIndex, 'NumTrain=', numTrain, 'NumValid=', numValid, 'NumTest=', numTest
self.X_train_DataSample.extend(li[:numTrain])
self.X_valid_DataSample.extend(li[numTrain : numTrain+numValid])
self.X_test_DataSample.extend(li[numTrain + numValid :])
self.y_train_cluster.extend([clusterIndex] * numTrain)
self.y_valid_cluster.extend([clusterIndex] * numValid)
self.y_test_cluster.extend([clusterIndex] * (len(li) - numTrain - numValid))
self.y_train_label.extend([labelIndices] * numTrain)
self.y_valid_label.extend([labelIndices] * numValid)
self.y_test_label.extend([labelIndices] * (len(li) - numTrain - numValid))
self.X_train_rawText = [i.rawText for i in self.X_train_DataSample]
self.X_valid_rawText = [i.rawText for i in self.X_valid_DataSample]
self.X_test_rawText = [i.rawText for i in self.X_test_DataSample]