def readData1(self, root_data_dir, participant_index, *args, **kw): self.rawData, labelsList = self.diskDataToLiveData(root_data_dir) data2, jointsList, objectsList = self.convertToDict( self.rawData, 'testing', verbose=self.verbose) # extract a set of labels labels = list(set(labelsList)) labels.sort() logging.info('') # convert text labels into numbers labelNumsList = None for n, k in enumerate(labelsList): res = [m for m, l in enumerate(labels) if l == k] if n == 0: labelNumsList = np.array(res) else: labelNumsList = np.vstack([labelNumsList, res]) logging.info('shape of number labels:' + str(labelNumsList.shape)) uu, tmp = utils.transformTimeSeriesToSeq( labelNumsList, self.paramsDict['windowSize'], self.paramsDict['windowOffset'], False, False) data2NumLabels = uu logging.info('windowed number labels shape:' + str(data2NumLabels.shape)) # now that labels are in windowed form it is time to # assign them a text label again that describes them # the rule here is if the window appertains to the same label, # that label is assigned otherwise it is labelled as transition data2Labels = [] for j in data2NumLabels: numItems = list(set(j)) if len(numItems) == 1: l = labels[int(numItems[0])] data2Labels.append(l) else: # Another way to do this would be to label it according to 75% majority # This would decrease the region size of the transition blocks # which are currently dependant on windowSize data2Labels.append('transition') logging.info('windowed data labels compressed:' + str(len(data2Labels))) logging.info('') # create list of specific joints to be used jointsToUse = [] objectDict = dict() handDict = dict() for j in self.paramsDict['includeParts']: if j == 'object': for k in objectsList: if k != 'partner': objectDict[k] = (len(jointsToUse)) jointsToUse.append(k) elif 'hand' in j: handDict[j] = (len(jointsToUse)) jointsToUse.append(j) else: jointsToUse.append(j) combineObjects = len(objectDict) > 1 combineHands = len(handDict) > 1 logging.info(jointsToUse) logging.info(objectDict) logging.info(handDict) # concatenate data for all joints in a single vector logging.info('') dataVecAll = None for j in jointsToUse: if dataVecAll is None: dataVecAll = data2[j] else: dataVecAll = np.hstack([dataVecAll, data2[j]]) itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse) logging.info(dataVecAll.shape) logging.info(itemsPerJoint) self.itemsPerJoint = itemsPerJoint logging.info('') # it is now time to combine objects if multiple exist # self.featureSequence = ['object'] logging.info('') combinedObjs = None if combineObjects: logging.info('Combining Objects') for j in range(len(data2Labels)): # logging.info(data2Labels[j]) if len(data2Labels[j].split('_')) > 2: idxBase = objectDict[data2Labels[j].split('_') [2]] * itemsPerJoint else: idxBase = objectDict[objectDict.keys()[0]] * itemsPerJoint if combinedObjs is None: combinedObjs = dataVecAll[j, idxBase:idxBase + itemsPerJoint] else: combinedObjs = np.vstack([ combinedObjs, dataVecAll[j, idxBase:idxBase + itemsPerJoint] ]) logging.info(combinedObjs.shape) logging.info(dataVecAll.shape) logging.info('') # it is now time to combine hands if multiple exist combinedHands = None if combineHands and self.paramsDict['combineHands']: logging.info('Combining hands') self.handsCombined = True self.featureSequence.append('hand') for j in range(len(data2Labels)): if len(data2Labels[j].split('_')) > 2: idxBase = handDict[data2Labels[j].split('_')[3] + data2Labels[j].split('_')[4].capitalize()] * \ itemsPerJoint else: idxBase = handDict[handDict.keys()[0]] * itemsPerJoint if combinedHands is None: combinedHands = dataVecAll[j, idxBase:idxBase + itemsPerJoint] else: combinedHands = np.vstack([ combinedHands, dataVecAll[j, idxBase:idxBase + itemsPerJoint] ]) logging.info(dataVecAll.shape) logging.info(combinedHands.shape) else: self.handsCombined = False dataVecReq = None if combinedHands is not None: dataVecReq = combinedHands if combinedObjs is not None: if dataVecReq is None: dataVecReq = combinedObjs else: dataVecReq = np.hstack([dataVecReq, combinedObjs]) logging.info(jointsToUse) for j, item in enumerate(jointsToUse): if self.handsCombined: if item not in handDict and item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if dataVecReq is None: dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: dataVecReq = np.hstack([ dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint] ]) else: if item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if dataVecReq is None: dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: dataVecReq = np.hstack([ dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint] ]) logging.info(dataVecReq.shape) logging.info(len(data2Labels)) logging.info('') self.dataVec = copy.deepcopy(dataVecReq) data2ShortLabels = [] for j in data2Labels: splitLabel = j.split('_') slabel = ('_'.join(splitLabel[:2])) if splitLabel[0] == 'push' or splitLabel[0] == 'pull': if splitLabel[-1] == 'no': add = splitLabel[-2] else: add = splitLabel[-1] if add == 'left' and self.paramsDict['flip']: if splitLabel[0] == 'push': splitLabel[0] = 'pull' else: splitLabel[0] = 'push' slabel = ('_'.join(splitLabel[:2])) if self.paramsDict['sepRL']: slabel += '_' + add data2ShortLabels.append(slabel) self.data2Labels = copy.deepcopy(data2ShortLabels) if self.paramsDict['sepRL']: if 'pull_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index( 'pull_object') == 'pull_object_right' self.paramsDict['actionsAllowedList'].append( 'pull_object_left') if 'push_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index( 'push_object') == 'push_object_right' self.paramsDict['actionsAllowedList'].append( 'push_object_left') # remove labels which will not be trained listToDelete = [] for n in reversed(range(len(data2Labels))): if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \ 'no' in data2Labels[n]: listToDelete.append(n) dataVecReq = np.delete(dataVecReq, listToDelete, axis=0) npdata2ShortLabels = np.asarray(data2ShortLabels) npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0) # find left hand push and pull and label as pull and push respectively data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels) self.Y = dataVecReq self.L = data2ShortLabels # logging.info('\n'.join(data2Labels)) logging.info(self.Y.shape) logging.info(len(self.L)) # now that all joints are in the form of a window, time to create # all possible vectors to classify self.allDataDict = dict() self.allDataDict['Y'] = self.dataVec self.allDataDict['L'] = self.data2Labels listOfVectorsToClassify = self.listOfClassificationVectors( self.featureSequence, objectsList) for j in listOfVectorsToClassify: logging.info(j)
def readData(self, root_data_dir, participant_index, *args, **kw): self.rawData, labelsList = self.diskDataToLiveData(root_data_dir) data2, jointsList, objectsList = self.convertToDict( self.rawData, 'testing', verbose=self.verbose) logging.info('unique labels' + str(set(labelsList))) # extract a set of labels labels = list(set(labelsList)) labels.sort() logging.info('') # convert text labels into numbers labelNumsList = None for n, k in enumerate(labelsList): res = [m for m, l in enumerate(labels) if l == k] if n == 0: labelNumsList = np.array(res) else: labelNumsList = np.vstack([labelNumsList, res]) logging.info('shape of number labels:' + str(labelNumsList.shape)) uu, tmp = utils.transformTimeSeriesToSeq( labelNumsList, self.paramsDict['windowSize'], self.paramsDict['windowOffset'], False, False) data2NumLabels = uu logging.info('windowed number labels shape:' + str(data2NumLabels.shape)) # now that labels are in windowed form it is time to # assign them a text label again that describes them # the rule here is if the window appertains to the same label, # that label is assigned otherwise it is labelled as transition data2Labels = [] for j in data2NumLabels: numItems = list(set(j)) if len(numItems) == 1: l = labels[int(numItems[0])] data2Labels.append(l) else: # Another way to do this would be to label it according to 75% majority # This would decrease the region size of the transition blocks # which are currently dependant on windowSize data2Labels.append('transition') logging.info('after transition unique set ' + str(set(data2Labels))) logging.info('windowed data labels compressed: ' + str(len(data2Labels))) logging.info('') # create list of specific joints to be used jointsToUse = [] objectDict = dict() handDict = dict() for j in self.paramsDict['includeParts']: if j == 'object': for k in objectsList: if k != 'partner': objectDict[k] = (len(jointsToUse)) jointsToUse.append(k) elif 'hand' in j: handDict[j] = (len(jointsToUse)) jointsToUse.append(j) else: jointsToUse.append(j) combineObjects = len(objectDict) > 1 combineHands = len(handDict) > 1 logging.info(jointsToUse) logging.info(objectDict) logging.info(handDict) # concatenate data for all joints in a single vector logging.info('') dataVecAll = None for j in jointsToUse: if dataVecAll is None: dataVecAll = data2[j] else: dataVecAll = np.hstack([dataVecAll, data2[j]]) itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse) logging.info(dataVecAll.shape) logging.info(itemsPerJoint) self.itemsPerJoint = itemsPerJoint logging.info('') # ------------------------------------------------------------------ # it is now time to combine objects if multiple exist # logging.info('') self.featureSequence = [] combinedObjs = dict() if combineObjects and 'object' in self.paramsDict['includeParts']: self.featureSequence.append('object') logging.info('Combining Objects') for n in objectDict: idxBase = objectDict[n] * itemsPerJoint combinedObjs[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint] logging.info(combinedObjs[n].shape) logging.info(dataVecAll.shape) logging.info('') # it is now time to combine hands if multiple exist combinedHands = dict() if combineHands and self.paramsDict['combineHands'] and \ len([s for s in self.paramsDict['includeParts'] if 'hand' in s]) > 0: logging.info('Combining hands') self.handsCombined = True self.featureSequence.append('hand') for n in handDict: idxBase = handDict[n] * itemsPerJoint combinedHands[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint] logging.info(combinedHands[n].shape) logging.info(dataVecAll.shape) else: self.handsCombined = False logging.info(jointsToUse) otherJoints = None for j, item in enumerate(jointsToUse): if self.handsCombined: if item not in handDict and item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if otherJoints is None: otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: otherJoints = np.hstack([ otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint] ]) else: if item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if otherJoints is None: otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: otherJoints = np.hstack([ otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint] ]) if otherJoints is not None: logging.info(otherJoints.shape) self.listOfVectorsToClassify = [] for j in self.featureSequence: if j == 'object': for k in objectsList: if k != 'partner': self.listOfVectorsToClassify.append([k]) elif 'hand' in j: if self.handsCombined: a = copy.deepcopy(self.listOfVectorsToClassify) b = copy.deepcopy(self.listOfVectorsToClassify) if len(self.listOfVectorsToClassify) > 0: for l, m in enumerate(self.listOfVectorsToClassify): a[l].append('handLeft') b[l].append('handRight') self.listOfVectorsToClassify = a + b else: self.listOfVectorsToClassify.append(['handLeft']) self.listOfVectorsToClassify.append(['handRight']) else: for l, m in enumerate(self.listOfVectorsToClassify): self.listOfVectorsToClassify[l].append(j) else: for l, m in enumerate(self.listOfVectorsToClassify): self.listOfVectorsToClassify[l].append(j) logging.info('Vectors to Classify:') for j in self.listOfVectorsToClassify: logging.info("\t" + str(j)) dataVecReq = None objSection = None if combinedObjs: objSection = None for j in self.listOfVectorsToClassify: logging.info(str(j[0])) if objSection is None: objSection = combinedObjs[j[0]] else: objSection = np.vstack([objSection, combinedObjs[j[0]]]) dataVecReq = objSection logging.info(str(objSection.shape)) handsSection = None if combinedHands: for j in self.listOfVectorsToClassify: for l in j: if 'hand' in l: if handsSection is None: handsSection = combinedHands[l] else: handsSection = np.vstack( [handsSection, combinedHands[l]]) if dataVecReq is None: dataVecReq = handsSection else: dataVecReq = np.hstack([dataVecReq, handsSection]) logging.info(str(handsSection.shape)) othersSection = None if otherJoints is not None: for j in self.listOfVectorsToClassify: logging.info(str(j[:])) if othersSection is None: othersSection = otherJoints else: othersSection = np.vstack([othersSection, otherJoints]) if dataVecReq is None: dataVecReq = othersSection else: dataVecReq = np.hstack([dataVecReq, othersSection]) logging.info(str(dataVecReq.shape)) del handsSection, othersSection, objSection, combinedHands, combinedObjs, otherJoints # Also augment the labels list data2LabelsAugment = [] for j in self.listOfVectorsToClassify: data2LabelsAugment.append([]) for j in data2Labels: splitLabel = j.split('_') action = '_'.join(splitLabel[:2]) if len(splitLabel) > 2: obj = splitLabel[2] hand = splitLabel[4] if combineHands: handSubList = [ k for k in self.listOfVectorsToClassify if 'hand' + hand.capitalize() in k ] if combineObjects: vec = [f for f in handSubList if obj in f][0] else: vec = handSubList[0] else: vec = [ f for f in self.listOfVectorsToClassify if obj in f ][0] # logging.info(data2Labels.index(j), vec) # printStr = '' for n, k in enumerate(self.listOfVectorsToClassify): if vec == k: data2LabelsAugment[n].append(action) # printStr += action + '\t' # else: data2LabelsAugment[n].append('idle') # printStr += '\tidle' # logging.info(data2LabelsAugment[n][-1],) # print else: obj = '' hand = '' printStr = '' for n, k in enumerate(self.listOfVectorsToClassify): # printStr += action + '\t' data2LabelsAugment[n].append(action) # logging.info(data2LabelsAugment[n][-1],) # print # logging.info(action, obj, hand) # logging.info('---------------------') # logging.info('before augment', set(data2Labels)) data2Labels = [] for j in data2LabelsAugment: data2Labels += j # logging.info('after augment', set(data2Labels) logging.info('labels ' + str(len(data2Labels))) logging.info('data ' + str(dataVecReq.shape)) self.allDataDict = dict() self.allDataDict['Y'] = copy.deepcopy(dataVecReq) self.allDataDict['L'] = copy.deepcopy(data2Labels) # --------------------------------------------------------------------------------- data2ShortLabels = [] for j in data2Labels: splitLabel = j.split('_') slabel = ('_'.join(splitLabel[:2])) if splitLabel[0] == 'push' or splitLabel[0] == 'pull': if splitLabel[-1] == 'no': add = splitLabel[-2] else: add = splitLabel[-1] if add == 'left' and self.paramsDict['flip']: if splitLabel[0] == 'push': splitLabel[0] = 'pull' else: splitLabel[0] = 'push' slabel = ('_'.join(splitLabel[:2])) if self.paramsDict['sepRL']: slabel += '_' + add data2ShortLabels.append(slabel) self.data2Labels = copy.deepcopy(data2ShortLabels) logging.info('shortLabels len ' + str(set(self.data2Labels))) if self.paramsDict['sepRL']: if 'pull_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index( 'pull_object') == 'pull_object_right' self.paramsDict['actionsAllowedList'].append( 'pull_object_left') if 'push_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index( 'push_object') == 'push_object_right' self.paramsDict['actionsAllowedList'].append( 'push_object_left') # remove labels which will not be trained logging.info('actions allowed: ' + str(self.paramsDict['actionsAllowedList'])) listToDelete = [] for n in reversed(range(len(data2Labels))): if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \ 'no' in data2Labels[n]: listToDelete.append(n) dataVecReq = np.delete(dataVecReq, listToDelete, axis=0) npdata2ShortLabels = np.asarray(data2ShortLabels) npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0) # find left hand push and pull and label as pull and push respectively data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels) self.Y = dataVecReq self.L = data2ShortLabels # logging.info('\n'.join(data2Labels)) logging.info(self.Y.shape) logging.info(len(self.L))
def convertToDict(self, rawData, mode, verbose): data = dict() firstPass = True jointsList = [] objectsList = [] # logging.info('*******************') # for j in self.paramsDict: # logging.info(j, self.paramsDict[j] # logging.info('*******************') for t in rawData: # parse skeleton data which has 9 sections by (x,y,z) for i in range(self.numJoints): a = i * 4 # if t[a] == 'shoulderCenter': # t[a] = 'chest' if firstPass: data[t[a]] = [None] data[t[a]] = (np.array( [float(t[a + 1]), float(t[a + 2]), float(t[a + 3])])) jointsList.append(t[a]) else: arr = np.array( [float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if data[t[a]] is not None: data[t[a]] = np.vstack((data[t[a]], arr)) else: data[t[a]] = arr currIdx = (self.numJoints * 4 - 1) numObjs = (len(t) - currIdx) / 5 for i in range(numObjs): a = currIdx + 1 + (i * 5) if t[a] in data: arr = np.array( [float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if data[t[a]] is not None: data[t[a]] = np.vstack((data[t[a]], arr)) else: data[t[a]] = arr else: data[t[a]] = [None] data[t[a]] = np.array( [float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if mode == 'testing' or (mode != 'testing' and t[a + 4] == '1'): objectsList.append(t[a]) firstPass = False if verbose: logging.info('data has length = ' + str(len(data)) + ' joints') logging.info('each joint has an array of shape ' + str(data['head'].shape)) if self.paramsDict['filterData'] or 'vel' in self.paramsDict['components'] or \ 'acc' in self.paramsDict['components']: if verbose: logging.info('Filtering data with hamming window of size ' + str(self.paramsDict['filterWindow'])) for j in data.keys(): t1 = utils.smooth1D(data[j][:, 0], self.paramsDict['filterWindow']) t2 = utils.smooth1D(data[j][:, 1], self.paramsDict['filterWindow']) t3 = utils.smooth1D(data[j][:, 2], self.paramsDict['filterWindow']) data[j] = np.hstack([t1[:, None], t2[:, None], t3[:, None]]) if verbose: logging.info('data has length = ' + str(len(data)) + ' joints') logging.info('each joint has an array of shape ' + str(data['head'].shape)) # convert data and number labels into windows. # data is still in the form of a dictionary with the joints/objects as keys of the dict # Text labels contained in labels if verbose: logging.info('') noY = mode != 'testing' if mode == 'testing': offset = self.paramsDict['windowOffset'] else: offset = 1 data2 = dict() printExplanation = True for num, key in enumerate(data): data2[key] = None xx, yy = utils.transformTimeSeriesToSeq( data[key], timeWindow=self.paramsDict['windowSize'], offset=offset, normalised=self.paramsDict['normaliseWindow'], reduced=self.paramsDict['reduce'], noY=noY) if self.paramsDict['thresholdMovement'] or 'vel' in self.paramsDict['components'] or 'acc' in \ self.paramsDict['components']: winSize = xx.shape[1] / 3 g = xx.size / winSize xxshape1 = xx.shape[0] xxshape2 = xx.shape[1] flatxx = xx.flatten() f = flatxx.reshape([g, winSize]) xx = f.reshape([xxshape1, xxshape2]) if self.paramsDict['thresholdMovement']: if printExplanation and verbose: logging.info('thresholding movement <' + str(self.paramsDict['moveThresh'])) ranges = np.ptp(f, axis=1) a = ranges < self.paramsDict['moveThresh'] b = ranges > -self.paramsDict['moveThresh'] res = list(np.where(np.logical_and(a, b))[0]) if self.paramsDict['normaliseWindow']: f[res] = 0 else: for ll in res: f[ll] = f[ll][0] if 'vel' in self.paramsDict['components']: if printExplanation and verbose: logging.info('Adding velocity to the feature vector') xxvel = np.diff(f) xxvel = xxvel.reshape([xxshape1, xxshape2 - 3]) xx = np.hstack([xx, xxvel]) if 'acc' in self.paramsDict['components']: if printExplanation and verbose: logging.info( 'Adding acceleration to the feature vector') xxacc = np.diff(f, n=2) xxacc = xxacc.reshape([xxshape1, xxshape2 - 6]) xx = np.hstack([xx, xxacc]) data2[key] = xx printExplanation = False if verbose: logging.info('data has length = ' + str(len(data2)) + ' joints') logging.info('each joint has an array of shape ' + str(data2['head'].shape)) return data2, jointsList, objectsList
def readData(self, root_data_dir, participant_index, *args, **kw): self.rawData, labelsList = self.diskDataToLiveData(root_data_dir) data2, jointsList, objectsList = self.convertToDict(self.rawData, 'testing', verbose=self.verbose) logging.info('unique labels' + str(set(labelsList))) # extract a set of labels labels = list(set(labelsList)) labels.sort() logging.info('') # convert text labels into numbers labelNumsList = None for n, k in enumerate(labelsList): res = [m for m, l in enumerate(labels) if l == k] if n == 0: labelNumsList = np.array(res) else: labelNumsList = np.vstack([labelNumsList, res]) logging.info('shape of number labels:' +str(labelNumsList.shape)) uu, tmp = utils.transformTimeSeriesToSeq(labelNumsList, self.paramsDict['windowSize'], self.paramsDict['windowOffset'], False, False) data2NumLabels = uu logging.info('windowed number labels shape:' + str(data2NumLabels.shape)) # now that labels are in windowed form it is time to # assign them a text label again that describes them # the rule here is if the window appertains to the same label, # that label is assigned otherwise it is labelled as transition data2Labels = [] for j in data2NumLabels: numItems = list(set(j)) if len(numItems) == 1: l = labels[int(numItems[0])] data2Labels.append(l) else: # Another way to do this would be to label it according to 75% majority # This would decrease the region size of the transition blocks # which are currently dependant on windowSize data2Labels.append('transition') logging.info('after transition unique set ' + str(set(data2Labels))) logging.info('windowed data labels compressed: ' + str(len(data2Labels))) logging.info('') # create list of specific joints to be used jointsToUse = [] objectDict = dict() handDict = dict() for j in self.paramsDict['includeParts']: if j == 'object': for k in objectsList: if k != 'partner': objectDict[k] = (len(jointsToUse)) jointsToUse.append(k) elif 'hand' in j: handDict[j] = (len(jointsToUse)) jointsToUse.append(j) else: jointsToUse.append(j) combineObjects = len(objectDict) > 1 combineHands = len(handDict) > 1 logging.info(jointsToUse) logging.info(objectDict) logging.info(handDict) # concatenate data for all joints in a single vector logging.info('') dataVecAll = None for j in jointsToUse: if dataVecAll is None: dataVecAll = data2[j] else: dataVecAll = np.hstack([dataVecAll, data2[j]]) itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse) logging.info(dataVecAll.shape) logging.info(itemsPerJoint) self.itemsPerJoint = itemsPerJoint logging.info('') # ------------------------------------------------------------------ # it is now time to combine objects if multiple exist # logging.info('') self.featureSequence = [] combinedObjs = dict() if combineObjects and 'object' in self.paramsDict['includeParts']: self.featureSequence.append('object') logging.info('Combining Objects') for n in objectDict: idxBase = objectDict[n] * itemsPerJoint combinedObjs[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint] logging.info(combinedObjs[n].shape) logging.info(dataVecAll.shape) logging.info('') # it is now time to combine hands if multiple exist combinedHands = dict() if combineHands and self.paramsDict['combineHands'] and \ len([s for s in self.paramsDict['includeParts'] if 'hand' in s]) > 0: logging.info('Combining hands') self.handsCombined = True self.featureSequence.append('hand') for n in handDict: idxBase = handDict[n] * itemsPerJoint combinedHands[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint] logging.info(combinedHands[n].shape) logging.info(dataVecAll.shape) else: self.handsCombined = False logging.info(jointsToUse) otherJoints = None for j, item in enumerate(jointsToUse): if self.handsCombined: if item not in handDict and item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if otherJoints is None: otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: otherJoints = np.hstack([otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint]]) else: if item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if otherJoints is None: otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: otherJoints = np.hstack([otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint]]) if otherJoints is not None: logging.info(otherJoints.shape) self.listOfVectorsToClassify = [] for j in self.featureSequence: if j == 'object': for k in objectsList: if k != 'partner': self.listOfVectorsToClassify.append([k]) elif 'hand' in j: if self.handsCombined: a = copy.deepcopy(self.listOfVectorsToClassify) b = copy.deepcopy(self.listOfVectorsToClassify) if len(self.listOfVectorsToClassify) > 0: for l, m in enumerate(self.listOfVectorsToClassify): a[l].append('handLeft') b[l].append('handRight') self.listOfVectorsToClassify = a + b else: self.listOfVectorsToClassify.append(['handLeft']) self.listOfVectorsToClassify.append(['handRight']) else: for l, m in enumerate(self.listOfVectorsToClassify): self.listOfVectorsToClassify[l].append(j) else: for l, m in enumerate(self.listOfVectorsToClassify): self.listOfVectorsToClassify[l].append(j) logging.info('Vectors to Classify:') for j in self.listOfVectorsToClassify: logging.info("\t" + str(j)) dataVecReq = None objSection = None if combinedObjs: objSection = None for j in self.listOfVectorsToClassify: logging.info(str(j[0])) if objSection is None: objSection = combinedObjs[j[0]] else: objSection = np.vstack([objSection, combinedObjs[j[0]]]) dataVecReq = objSection logging.info(str(objSection.shape)) handsSection = None if combinedHands: for j in self.listOfVectorsToClassify: for l in j: if 'hand' in l: if handsSection is None: handsSection = combinedHands[l] else: handsSection = np.vstack([handsSection, combinedHands[l]]) if dataVecReq is None: dataVecReq = handsSection else: dataVecReq = np.hstack([dataVecReq, handsSection]) logging.info(str(handsSection.shape)) othersSection = None if otherJoints is not None: for j in self.listOfVectorsToClassify: logging.info(str(j[:])) if othersSection is None: othersSection = otherJoints else: othersSection = np.vstack([othersSection, otherJoints]) if dataVecReq is None: dataVecReq = othersSection else: dataVecReq = np.hstack([dataVecReq, othersSection]) logging.info(str(dataVecReq.shape)) del handsSection, othersSection, objSection, combinedHands, combinedObjs, otherJoints # Also augment the labels list data2LabelsAugment = [] for j in self.listOfVectorsToClassify: data2LabelsAugment.append([]) for j in data2Labels: splitLabel = j.split('_') action = '_'.join(splitLabel[:2]) if len(splitLabel) > 2: obj = splitLabel[2] hand = splitLabel[4] if combineHands: handSubList = [k for k in self.listOfVectorsToClassify if 'hand' + hand.capitalize() in k] if combineObjects: vec = [f for f in handSubList if obj in f][0] else: vec = handSubList[0] else: vec = [f for f in self.listOfVectorsToClassify if obj in f][0] # logging.info(data2Labels.index(j), vec) # printStr = '' for n, k in enumerate(self.listOfVectorsToClassify): if vec == k: data2LabelsAugment[n].append(action) # printStr += action + '\t' # else: data2LabelsAugment[n].append('idle') # printStr += '\tidle' # logging.info(data2LabelsAugment[n][-1],) # print else: obj = '' hand = '' printStr = '' for n, k in enumerate(self.listOfVectorsToClassify): # printStr += action + '\t' data2LabelsAugment[n].append(action) # logging.info(data2LabelsAugment[n][-1],) # print # logging.info(action, obj, hand) # logging.info('---------------------') # logging.info('before augment', set(data2Labels)) data2Labels = [] for j in data2LabelsAugment: data2Labels += j # logging.info('after augment', set(data2Labels) logging.info('labels ' + str(len(data2Labels))) logging.info('data ' + str(dataVecReq.shape)) self.allDataDict = dict() self.allDataDict['Y'] = copy.deepcopy(dataVecReq) self.allDataDict['L'] = copy.deepcopy(data2Labels) # --------------------------------------------------------------------------------- data2ShortLabels = [] for j in data2Labels: splitLabel = j.split('_') slabel = ('_'.join(splitLabel[:2])) if splitLabel[0] == 'push' or splitLabel[0] == 'pull': if splitLabel[-1] == 'no': add = splitLabel[-2] else: add = splitLabel[-1] if add == 'left' and self.paramsDict['flip']: if splitLabel[0] == 'push': splitLabel[0] = 'pull' else: splitLabel[0] = 'push' slabel = ('_'.join(splitLabel[:2])) if self.paramsDict['sepRL']: slabel += '_' + add data2ShortLabels.append(slabel) self.data2Labels = copy.deepcopy(data2ShortLabels) logging.info('shortLabels len ' + str(set(self.data2Labels))) if self.paramsDict['sepRL']: if 'pull_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index('pull_object') == 'pull_object_right' self.paramsDict['actionsAllowedList'].append('pull_object_left') if 'push_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index('push_object') == 'push_object_right' self.paramsDict['actionsAllowedList'].append('push_object_left') # remove labels which will not be trained logging.info('actions allowed: ' + str(self.paramsDict['actionsAllowedList'])) listToDelete = [] for n in reversed(range(len(data2Labels))): if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \ 'no' in data2Labels[n]: listToDelete.append(n) dataVecReq = np.delete(dataVecReq, listToDelete, axis=0) npdata2ShortLabels = np.asarray(data2ShortLabels) npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0) # find left hand push and pull and label as pull and push respectively data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels) self.Y = dataVecReq self.L = data2ShortLabels # logging.info('\n'.join(data2Labels)) logging.info(self.Y.shape) logging.info(len(self.L))
def readData1(self, root_data_dir, participant_index, *args, **kw): self.rawData, labelsList = self.diskDataToLiveData(root_data_dir) data2, jointsList, objectsList = self.convertToDict(self.rawData, 'testing', verbose=self.verbose) # extract a set of labels labels = list(set(labelsList)) labels.sort() logging.info('') # convert text labels into numbers labelNumsList = None for n, k in enumerate(labelsList): res = [m for m, l in enumerate(labels) if l == k] if n == 0: labelNumsList = np.array(res) else: labelNumsList = np.vstack([labelNumsList, res]) logging.info('shape of number labels:' + str(labelNumsList.shape)) uu, tmp = utils.transformTimeSeriesToSeq(labelNumsList, self.paramsDict['windowSize'], self.paramsDict['windowOffset'], False, False) data2NumLabels = uu logging.info('windowed number labels shape:' + str(data2NumLabels.shape)) # now that labels are in windowed form it is time to # assign them a text label again that describes them # the rule here is if the window appertains to the same label, # that label is assigned otherwise it is labelled as transition data2Labels = [] for j in data2NumLabels: numItems = list(set(j)) if len(numItems) == 1: l = labels[int(numItems[0])] data2Labels.append(l) else: # Another way to do this would be to label it according to 75% majority # This would decrease the region size of the transition blocks # which are currently dependant on windowSize data2Labels.append('transition') logging.info('windowed data labels compressed:' + str(len(data2Labels))) logging.info('') # create list of specific joints to be used jointsToUse = [] objectDict = dict() handDict = dict() for j in self.paramsDict['includeParts']: if j == 'object': for k in objectsList: if k != 'partner': objectDict[k] = (len(jointsToUse)) jointsToUse.append(k) elif 'hand' in j: handDict[j] = (len(jointsToUse)) jointsToUse.append(j) else: jointsToUse.append(j) combineObjects = len(objectDict) > 1 combineHands = len(handDict) > 1 logging.info(jointsToUse) logging.info(objectDict) logging.info(handDict) # concatenate data for all joints in a single vector logging.info('') dataVecAll = None for j in jointsToUse: if dataVecAll is None: dataVecAll = data2[j] else: dataVecAll = np.hstack([dataVecAll, data2[j]]) itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse) logging.info(dataVecAll.shape) logging.info(itemsPerJoint) self.itemsPerJoint = itemsPerJoint logging.info('') # it is now time to combine objects if multiple exist # self.featureSequence = ['object'] logging.info('') combinedObjs = None if combineObjects: logging.info('Combining Objects') for j in range(len(data2Labels)): # logging.info(data2Labels[j]) if len(data2Labels[j].split('_')) > 2: idxBase = objectDict[data2Labels[j].split('_')[2]] * itemsPerJoint else: idxBase = objectDict[objectDict.keys()[0]] * itemsPerJoint if combinedObjs is None: combinedObjs = dataVecAll[j, idxBase:idxBase + itemsPerJoint] else: combinedObjs = np.vstack([combinedObjs, dataVecAll[j, idxBase:idxBase + itemsPerJoint]]) logging.info(combinedObjs.shape) logging.info(dataVecAll.shape) logging.info('') # it is now time to combine hands if multiple exist combinedHands = None if combineHands and self.paramsDict['combineHands']: logging.info('Combining hands') self.handsCombined = True self.featureSequence.append('hand') for j in range(len(data2Labels)): if len(data2Labels[j].split('_')) > 2: idxBase = handDict[data2Labels[j].split('_')[3] + data2Labels[j].split('_')[4].capitalize()] * \ itemsPerJoint else: idxBase = handDict[handDict.keys()[0]] * itemsPerJoint if combinedHands is None: combinedHands = dataVecAll[j, idxBase:idxBase + itemsPerJoint] else: combinedHands = np.vstack([combinedHands, dataVecAll[j, idxBase:idxBase + itemsPerJoint]]) logging.info(dataVecAll.shape) logging.info(combinedHands.shape) else: self.handsCombined = False dataVecReq = None if combinedHands is not None: dataVecReq = combinedHands if combinedObjs is not None: if dataVecReq is None: dataVecReq = combinedObjs else: dataVecReq = np.hstack([dataVecReq, combinedObjs]) logging.info(jointsToUse) for j, item in enumerate(jointsToUse): if self.handsCombined: if item not in handDict and item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if dataVecReq is None: dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: dataVecReq = np.hstack([dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint]]) else: if item not in objectDict: self.featureSequence.append(item) idxBase = j * itemsPerJoint if dataVecReq is None: dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint] else: dataVecReq = np.hstack([dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint]]) logging.info(dataVecReq.shape) logging.info(len(data2Labels)) logging.info('') self.dataVec = copy.deepcopy(dataVecReq) data2ShortLabels = [] for j in data2Labels: splitLabel = j.split('_') slabel = ('_'.join(splitLabel[:2])) if splitLabel[0] == 'push' or splitLabel[0] == 'pull': if splitLabel[-1] == 'no': add = splitLabel[-2] else: add = splitLabel[-1] if add == 'left' and self.paramsDict['flip']: if splitLabel[0] == 'push': splitLabel[0] = 'pull' else: splitLabel[0] = 'push' slabel = ('_'.join(splitLabel[:2])) if self.paramsDict['sepRL']: slabel += '_' + add data2ShortLabels.append(slabel) self.data2Labels = copy.deepcopy(data2ShortLabels) if self.paramsDict['sepRL']: if 'pull_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index('pull_object') == 'pull_object_right' self.paramsDict['actionsAllowedList'].append('pull_object_left') if 'push_object' in self.paramsDict['actionsAllowedList']: self.paramsDict['actionsAllowedList'].index('push_object') == 'push_object_right' self.paramsDict['actionsAllowedList'].append('push_object_left') # remove labels which will not be trained listToDelete = [] for n in reversed(range(len(data2Labels))): if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \ 'no' in data2Labels[n]: listToDelete.append(n) dataVecReq = np.delete(dataVecReq, listToDelete, axis=0) npdata2ShortLabels = np.asarray(data2ShortLabels) npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0) # find left hand push and pull and label as pull and push respectively data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels) self.Y = dataVecReq self.L = data2ShortLabels # logging.info('\n'.join(data2Labels)) logging.info(self.Y.shape) logging.info(len(self.L)) # now that all joints are in the form of a window, time to create # all possible vectors to classify self.allDataDict = dict() self.allDataDict['Y'] = self.dataVec self.allDataDict['L'] = self.data2Labels listOfVectorsToClassify = self.listOfClassificationVectors(self.featureSequence, objectsList) for j in listOfVectorsToClassify: logging.info(j)
def convertToDict(self, rawData, mode, verbose): data = dict() firstPass = True jointsList = [] objectsList = [] # logging.info('*******************') # for j in self.paramsDict: # logging.info(j, self.paramsDict[j] # logging.info('*******************') for t in rawData: # parse skeleton data which has 9 sections by (x,y,z) for i in range(self.numJoints): a = i * 4 # if t[a] == 'shoulderCenter': # t[a] = 'chest' if firstPass: data[t[a]] = [None] data[t[a]] = (np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])])) jointsList.append(t[a]) else: arr = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if data[t[a]] is not None: data[t[a]] = np.vstack((data[t[a]], arr)) else: data[t[a]] = arr currIdx = (self.numJoints * 4 - 1) numObjs = (len(t) - currIdx) / 5 for i in range(numObjs): a = currIdx + 1 + (i * 5) if t[a] in data: arr = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if data[t[a]] is not None: data[t[a]] = np.vstack((data[t[a]], arr)) else: data[t[a]] = arr else: data[t[a]] = [None] data[t[a]] = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]) if mode == 'testing' or (mode != 'testing' and t[a+4] == '1'): objectsList.append(t[a]) firstPass = False if verbose: logging.info('data has length = ' + str(len(data)) + ' joints') logging.info('each joint has an array of shape ' + str(data['head'].shape)) if self.paramsDict['filterData'] or 'vel' in self.paramsDict['components'] or \ 'acc' in self.paramsDict['components']: if verbose: logging.info('Filtering data with hamming window of size ' + str(self.paramsDict['filterWindow'])) for j in data.keys(): t1 = utils.smooth1D(data[j][:, 0], self.paramsDict['filterWindow']) t2 = utils.smooth1D(data[j][:, 1], self.paramsDict['filterWindow']) t3 = utils.smooth1D(data[j][:, 2], self.paramsDict['filterWindow']) data[j] = np.hstack([t1[:, None], t2[:, None], t3[:, None]]) if verbose: logging.info('data has length = ' + str(len(data)) + ' joints') logging.info('each joint has an array of shape ' + str(data['head'].shape)) # convert data and number labels into windows. # data is still in the form of a dictionary with the joints/objects as keys of the dict # Text labels contained in labels if verbose: logging.info('') noY = mode != 'testing' if mode == 'testing': offset = self.paramsDict['windowOffset'] else: offset = 1 data2 = dict() printExplanation = True for num, key in enumerate(data): data2[key] = None xx, yy = utils.transformTimeSeriesToSeq(data[key], timeWindow=self.paramsDict['windowSize'], offset=offset, normalised=self.paramsDict['normaliseWindow'], reduced=self.paramsDict['reduce'], noY=noY) if self.paramsDict['thresholdMovement'] or 'vel' in self.paramsDict['components'] or 'acc' in \ self.paramsDict['components']: winSize = xx.shape[1] / 3 g = xx.size / winSize xxshape1 = xx.shape[0] xxshape2 = xx.shape[1] flatxx = xx.flatten() f = flatxx.reshape([g, winSize]) xx = f.reshape([xxshape1, xxshape2]) if self.paramsDict['thresholdMovement']: if printExplanation and verbose: logging.info('thresholding movement <' + str(self.paramsDict['moveThresh'])) ranges = np.ptp(f, axis=1) a = ranges < self.paramsDict['moveThresh'] b = ranges > -self.paramsDict['moveThresh'] res = list(np.where(np.logical_and(a, b))[0]) if self.paramsDict['normaliseWindow']: f[res] = 0 else: for ll in res: f[ll] = f[ll][0] if 'vel' in self.paramsDict['components']: if printExplanation and verbose: logging.info('Adding velocity to the feature vector') xxvel = np.diff(f) xxvel = xxvel.reshape([xxshape1, xxshape2 - 3]) xx = np.hstack([xx, xxvel]) if 'acc' in self.paramsDict['components']: if printExplanation and verbose: logging.info('Adding acceleration to the feature vector') xxacc = np.diff(f, n=2) xxacc = xxacc.reshape([xxshape1, xxshape2 - 6]) xx = np.hstack([xx, xxacc]) data2[key] = xx printExplanation = False if verbose: logging.info('data has length = ' + str(len(data2)) + ' joints') logging.info('each joint has an array of shape ' + str(data2['head'].shape)) return data2, jointsList, objectsList
# #----------- Train a standard model # m1=SAM.SAM_Core.LFM() # m1.store(observed={'Y':Y1[0:Ntr,:]}, inputs=X1[0:Ntr,:], Q=None, kernel=None, num_inducing=num_inducing) # m1.learn() # ret = m1.visualise() # y_pred_standard = m1.pattern_completion(X1[Ntr:,:])[0] # pb.figure() # pb.plot(X1[Ntr:,:],Y1[Ntr:,:], 'x-') # pb.plot(X1[Ntr:,:],y_pred_standard, 'ro-') # pb.legend(('True','Pred')) # pb.title('Standard GP') # # --------------------------------------- # Create transformed data (autoregressive dataset) ws = 10 # Windowsize xx, yy = autoregressive.transformTimeSeriesToSeq(Y1, ws) # uu,tmp = transformTimeSeriesToSeq(U1, ws) # Test the above: np.sin(uu) - xx # uu = yy**2 -2*yy + 5 + np.random.randn(*yy.shape) * 0.005 U1 = Y1**2 - 2 * Y1 + 5 + np.random.randn(*Y1.shape) * 0.005 uu, tmp = autoregressive.transformTimeSeriesToSeq(U1, ws) Xtr = xx[0:Ntr, :] Xts = xx[Ntr:, :] Ytr = yy[0:Ntr, :] Yts = yy[Ntr:, :] Utr = uu[0:Ntr, :] Uts = uu[Ntr:, :]