def readData1(self, root_data_dir, participant_index, *args, **kw):
self.rawData, labelsList = self.diskDataToLiveData(root_data_dir)
data2, jointsList, objectsList = self.convertToDict(
self.rawData, 'testing', verbose=self.verbose)
# extract a set of labels
labels = list(set(labelsList))
labels.sort()
logging.info('')
# convert text labels into numbers
labelNumsList = None
for n, k in enumerate(labelsList):
res = [m for m, l in enumerate(labels) if l == k]
if n == 0:
labelNumsList = np.array(res)
else:
labelNumsList = np.vstack([labelNumsList, res])
logging.info('shape of number labels:' + str(labelNumsList.shape))
uu, tmp = utils.transformTimeSeriesToSeq(
labelNumsList, self.paramsDict['windowSize'],
self.paramsDict['windowOffset'], False, False)
data2NumLabels = uu
logging.info('windowed number labels shape:' +
str(data2NumLabels.shape))
# now that labels are in windowed form it is time to
# assign them a text label again that describes them
# the rule here is if the window appertains to the same label,
# that label is assigned otherwise it is labelled as transition
data2Labels = []
for j in data2NumLabels:
numItems = list(set(j))
if len(numItems) == 1:
l = labels[int(numItems[0])]
data2Labels.append(l)
else:
# Another way to do this would be to label it according to 75% majority
# This would decrease the region size of the transition blocks
# which are currently dependant on windowSize
data2Labels.append('transition')
logging.info('windowed data labels compressed:' +
str(len(data2Labels)))
logging.info('')
# create list of specific joints to be used
jointsToUse = []
objectDict = dict()
handDict = dict()
for j in self.paramsDict['includeParts']:
if j == 'object':
for k in objectsList:
if k != 'partner':
objectDict[k] = (len(jointsToUse))
jointsToUse.append(k)
elif 'hand' in j:
handDict[j] = (len(jointsToUse))
jointsToUse.append(j)
else:
jointsToUse.append(j)
combineObjects = len(objectDict) > 1
combineHands = len(handDict) > 1
logging.info(jointsToUse)
logging.info(objectDict)
logging.info(handDict)
# concatenate data for all joints in a single vector
logging.info('')
dataVecAll = None
for j in jointsToUse:
if dataVecAll is None:
dataVecAll = data2[j]
else:
dataVecAll = np.hstack([dataVecAll, data2[j]])
itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse)
logging.info(dataVecAll.shape)
logging.info(itemsPerJoint)
self.itemsPerJoint = itemsPerJoint
logging.info('')
# it is now time to combine objects if multiple exist
#
self.featureSequence = ['object']
logging.info('')
combinedObjs = None
if combineObjects:
logging.info('Combining Objects')
for j in range(len(data2Labels)):
# logging.info(data2Labels[j])
if len(data2Labels[j].split('_')) > 2:
idxBase = objectDict[data2Labels[j].split('_')
[2]] * itemsPerJoint
else:
idxBase = objectDict[objectDict.keys()[0]] * itemsPerJoint
if combinedObjs is None:
combinedObjs = dataVecAll[j,
idxBase:idxBase + itemsPerJoint]
else:
combinedObjs = np.vstack([
combinedObjs,
dataVecAll[j, idxBase:idxBase + itemsPerJoint]
])
logging.info(combinedObjs.shape)
logging.info(dataVecAll.shape)
logging.info('')
# it is now time to combine hands if multiple exist
combinedHands = None
if combineHands and self.paramsDict['combineHands']:
logging.info('Combining hands')
self.handsCombined = True
self.featureSequence.append('hand')
for j in range(len(data2Labels)):
if len(data2Labels[j].split('_')) > 2:
idxBase = handDict[data2Labels[j].split('_')[3] + data2Labels[j].split('_')[4].capitalize()] * \
itemsPerJoint
else:
idxBase = handDict[handDict.keys()[0]] * itemsPerJoint
if combinedHands is None:
combinedHands = dataVecAll[j,
idxBase:idxBase + itemsPerJoint]
else:
combinedHands = np.vstack([
combinedHands,
dataVecAll[j, idxBase:idxBase + itemsPerJoint]
])
logging.info(dataVecAll.shape)
logging.info(combinedHands.shape)
else:
self.handsCombined = False
dataVecReq = None
if combinedHands is not None:
dataVecReq = combinedHands
if combinedObjs is not None:
if dataVecReq is None:
dataVecReq = combinedObjs
else:
dataVecReq = np.hstack([dataVecReq, combinedObjs])
logging.info(jointsToUse)
for j, item in enumerate(jointsToUse):
if self.handsCombined:
if item not in handDict and item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if dataVecReq is None:
dataVecReq = dataVecAll[:, idxBase:idxBase +
itemsPerJoint]
else:
dataVecReq = np.hstack([
dataVecReq,
dataVecAll[:, idxBase:idxBase + itemsPerJoint]
])
else:
if item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if dataVecReq is None:
dataVecReq = dataVecAll[:, idxBase:idxBase +
itemsPerJoint]
else:
dataVecReq = np.hstack([
dataVecReq,
dataVecAll[:, idxBase:idxBase + itemsPerJoint]
])
logging.info(dataVecReq.shape)
logging.info(len(data2Labels))
logging.info('')
self.dataVec = copy.deepcopy(dataVecReq)
data2ShortLabels = []
for j in data2Labels:
splitLabel = j.split('_')
slabel = ('_'.join(splitLabel[:2]))
if splitLabel[0] == 'push' or splitLabel[0] == 'pull':
if splitLabel[-1] == 'no':
add = splitLabel[-2]
else:
add = splitLabel[-1]
if add == 'left' and self.paramsDict['flip']:
if splitLabel[0] == 'push':
splitLabel[0] = 'pull'
else:
splitLabel[0] = 'push'
slabel = ('_'.join(splitLabel[:2]))
if self.paramsDict['sepRL']:
slabel += '_' + add
data2ShortLabels.append(slabel)
self.data2Labels = copy.deepcopy(data2ShortLabels)
if self.paramsDict['sepRL']:
if 'pull_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index(
'pull_object') == 'pull_object_right'
self.paramsDict['actionsAllowedList'].append(
'pull_object_left')
if 'push_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index(
'push_object') == 'push_object_right'
self.paramsDict['actionsAllowedList'].append(
'push_object_left')
# remove labels which will not be trained
listToDelete = []
for n in reversed(range(len(data2Labels))):
if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \
'no' in data2Labels[n]:
listToDelete.append(n)
dataVecReq = np.delete(dataVecReq, listToDelete, axis=0)
npdata2ShortLabels = np.asarray(data2ShortLabels)
npdata2ShortLabels = np.delete(npdata2ShortLabels,
listToDelete,
axis=0)
# find left hand push and pull and label as pull and push respectively
data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels)
self.Y = dataVecReq
self.L = data2ShortLabels
# logging.info('\n'.join(data2Labels))
logging.info(self.Y.shape)
logging.info(len(self.L))
# now that all joints are in the form of a window, time to create
# all possible vectors to classify
self.allDataDict = dict()
self.allDataDict['Y'] = self.dataVec
self.allDataDict['L'] = self.data2Labels
listOfVectorsToClassify = self.listOfClassificationVectors(
self.featureSequence, objectsList)
for j in listOfVectorsToClassify:
logging.info(j)
def readData(self, root_data_dir, participant_index, *args, **kw):
self.rawData, labelsList = self.diskDataToLiveData(root_data_dir)
data2, jointsList, objectsList = self.convertToDict(
self.rawData, 'testing', verbose=self.verbose)
logging.info('unique labels' + str(set(labelsList)))
# extract a set of labels
labels = list(set(labelsList))
labels.sort()
logging.info('')
# convert text labels into numbers
labelNumsList = None
for n, k in enumerate(labelsList):
res = [m for m, l in enumerate(labels) if l == k]
if n == 0:
labelNumsList = np.array(res)
else:
labelNumsList = np.vstack([labelNumsList, res])
logging.info('shape of number labels:' + str(labelNumsList.shape))
uu, tmp = utils.transformTimeSeriesToSeq(
labelNumsList, self.paramsDict['windowSize'],
self.paramsDict['windowOffset'], False, False)
data2NumLabels = uu
logging.info('windowed number labels shape:' +
str(data2NumLabels.shape))
# now that labels are in windowed form it is time to
# assign them a text label again that describes them
# the rule here is if the window appertains to the same label,
# that label is assigned otherwise it is labelled as transition
data2Labels = []
for j in data2NumLabels:
numItems = list(set(j))
if len(numItems) == 1:
l = labels[int(numItems[0])]
data2Labels.append(l)
else:
# Another way to do this would be to label it according to 75% majority
# This would decrease the region size of the transition blocks
# which are currently dependant on windowSize
data2Labels.append('transition')
logging.info('after transition unique set ' + str(set(data2Labels)))
logging.info('windowed data labels compressed: ' +
str(len(data2Labels)))
logging.info('')
# create list of specific joints to be used
jointsToUse = []
objectDict = dict()
handDict = dict()
for j in self.paramsDict['includeParts']:
if j == 'object':
for k in objectsList:
if k != 'partner':
objectDict[k] = (len(jointsToUse))
jointsToUse.append(k)
elif 'hand' in j:
handDict[j] = (len(jointsToUse))
jointsToUse.append(j)
else:
jointsToUse.append(j)
combineObjects = len(objectDict) > 1
combineHands = len(handDict) > 1
logging.info(jointsToUse)
logging.info(objectDict)
logging.info(handDict)
# concatenate data for all joints in a single vector
logging.info('')
dataVecAll = None
for j in jointsToUse:
if dataVecAll is None:
dataVecAll = data2[j]
else:
dataVecAll = np.hstack([dataVecAll, data2[j]])
itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse)
logging.info(dataVecAll.shape)
logging.info(itemsPerJoint)
self.itemsPerJoint = itemsPerJoint
logging.info('')
# ------------------------------------------------------------------
# it is now time to combine objects if multiple exist
#
logging.info('')
self.featureSequence = []
combinedObjs = dict()
if combineObjects and 'object' in self.paramsDict['includeParts']:
self.featureSequence.append('object')
logging.info('Combining Objects')
for n in objectDict:
idxBase = objectDict[n] * itemsPerJoint
combinedObjs[n] = dataVecAll[:,
idxBase:idxBase + itemsPerJoint]
logging.info(combinedObjs[n].shape)
logging.info(dataVecAll.shape)
logging.info('')
# it is now time to combine hands if multiple exist
combinedHands = dict()
if combineHands and self.paramsDict['combineHands'] and \
len([s for s in self.paramsDict['includeParts'] if 'hand' in s]) > 0:
logging.info('Combining hands')
self.handsCombined = True
self.featureSequence.append('hand')
for n in handDict:
idxBase = handDict[n] * itemsPerJoint
combinedHands[n] = dataVecAll[:,
idxBase:idxBase + itemsPerJoint]
logging.info(combinedHands[n].shape)
logging.info(dataVecAll.shape)
else:
self.handsCombined = False
logging.info(jointsToUse)
otherJoints = None
for j, item in enumerate(jointsToUse):
if self.handsCombined:
if item not in handDict and item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if otherJoints is None:
otherJoints = dataVecAll[:, idxBase:idxBase +
itemsPerJoint]
else:
otherJoints = np.hstack([
otherJoints,
dataVecAll[:, idxBase:idxBase + itemsPerJoint]
])
else:
if item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if otherJoints is None:
otherJoints = dataVecAll[:, idxBase:idxBase +
itemsPerJoint]
else:
otherJoints = np.hstack([
otherJoints,
dataVecAll[:, idxBase:idxBase + itemsPerJoint]
])
if otherJoints is not None:
logging.info(otherJoints.shape)
self.listOfVectorsToClassify = []
for j in self.featureSequence:
if j == 'object':
for k in objectsList:
if k != 'partner':
self.listOfVectorsToClassify.append([k])
elif 'hand' in j:
if self.handsCombined:
a = copy.deepcopy(self.listOfVectorsToClassify)
b = copy.deepcopy(self.listOfVectorsToClassify)
if len(self.listOfVectorsToClassify) > 0:
for l, m in enumerate(self.listOfVectorsToClassify):
a[l].append('handLeft')
b[l].append('handRight')
self.listOfVectorsToClassify = a + b
else:
self.listOfVectorsToClassify.append(['handLeft'])
self.listOfVectorsToClassify.append(['handRight'])
else:
for l, m in enumerate(self.listOfVectorsToClassify):
self.listOfVectorsToClassify[l].append(j)
else:
for l, m in enumerate(self.listOfVectorsToClassify):
self.listOfVectorsToClassify[l].append(j)
logging.info('Vectors to Classify:')
for j in self.listOfVectorsToClassify:
logging.info("\t" + str(j))
dataVecReq = None
objSection = None
if combinedObjs:
objSection = None
for j in self.listOfVectorsToClassify:
logging.info(str(j[0]))
if objSection is None:
objSection = combinedObjs[j[0]]
else:
objSection = np.vstack([objSection, combinedObjs[j[0]]])
dataVecReq = objSection
logging.info(str(objSection.shape))
handsSection = None
if combinedHands:
for j in self.listOfVectorsToClassify:
for l in j:
if 'hand' in l:
if handsSection is None:
handsSection = combinedHands[l]
else:
handsSection = np.vstack(
[handsSection, combinedHands[l]])
if dataVecReq is None:
dataVecReq = handsSection
else:
dataVecReq = np.hstack([dataVecReq, handsSection])
logging.info(str(handsSection.shape))
othersSection = None
if otherJoints is not None:
for j in self.listOfVectorsToClassify:
logging.info(str(j[:]))
if othersSection is None:
othersSection = otherJoints
else:
othersSection = np.vstack([othersSection, otherJoints])
if dataVecReq is None:
dataVecReq = othersSection
else:
dataVecReq = np.hstack([dataVecReq, othersSection])
logging.info(str(dataVecReq.shape))
del handsSection, othersSection, objSection, combinedHands, combinedObjs, otherJoints
# Also augment the labels list
data2LabelsAugment = []
for j in self.listOfVectorsToClassify:
data2LabelsAugment.append([])
for j in data2Labels:
splitLabel = j.split('_')
action = '_'.join(splitLabel[:2])
if len(splitLabel) > 2:
obj = splitLabel[2]
hand = splitLabel[4]
if combineHands:
handSubList = [
k for k in self.listOfVectorsToClassify
if 'hand' + hand.capitalize() in k
]
if combineObjects:
vec = [f for f in handSubList if obj in f][0]
else:
vec = handSubList[0]
else:
vec = [
f for f in self.listOfVectorsToClassify if obj in f
][0]
# logging.info(data2Labels.index(j), vec)
# printStr = ''
for n, k in enumerate(self.listOfVectorsToClassify):
if vec == k:
data2LabelsAugment[n].append(action)
# printStr += action + '\t'
# else:
data2LabelsAugment[n].append('idle')
# printStr += '\tidle'
# logging.info(data2LabelsAugment[n][-1],)
# print
else:
obj = ''
hand = ''
printStr = ''
for n, k in enumerate(self.listOfVectorsToClassify):
# printStr += action + '\t'
data2LabelsAugment[n].append(action)
# logging.info(data2LabelsAugment[n][-1],)
# print
# logging.info(action, obj, hand)
# logging.info('---------------------')
# logging.info('before augment', set(data2Labels))
data2Labels = []
for j in data2LabelsAugment:
data2Labels += j
# logging.info('after augment', set(data2Labels)
logging.info('labels ' + str(len(data2Labels)))
logging.info('data ' + str(dataVecReq.shape))
self.allDataDict = dict()
self.allDataDict['Y'] = copy.deepcopy(dataVecReq)
self.allDataDict['L'] = copy.deepcopy(data2Labels)
# ---------------------------------------------------------------------------------
data2ShortLabels = []
for j in data2Labels:
splitLabel = j.split('_')
slabel = ('_'.join(splitLabel[:2]))
if splitLabel[0] == 'push' or splitLabel[0] == 'pull':
if splitLabel[-1] == 'no':
add = splitLabel[-2]
else:
add = splitLabel[-1]
if add == 'left' and self.paramsDict['flip']:
if splitLabel[0] == 'push':
splitLabel[0] = 'pull'
else:
splitLabel[0] = 'push'
slabel = ('_'.join(splitLabel[:2]))
if self.paramsDict['sepRL']:
slabel += '_' + add
data2ShortLabels.append(slabel)
self.data2Labels = copy.deepcopy(data2ShortLabels)
logging.info('shortLabels len ' + str(set(self.data2Labels)))
if self.paramsDict['sepRL']:
if 'pull_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index(
'pull_object') == 'pull_object_right'
self.paramsDict['actionsAllowedList'].append(
'pull_object_left')
if 'push_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index(
'push_object') == 'push_object_right'
self.paramsDict['actionsAllowedList'].append(
'push_object_left')
# remove labels which will not be trained
logging.info('actions allowed: ' +
str(self.paramsDict['actionsAllowedList']))
listToDelete = []
for n in reversed(range(len(data2Labels))):
if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \
'no' in data2Labels[n]:
listToDelete.append(n)
dataVecReq = np.delete(dataVecReq, listToDelete, axis=0)
npdata2ShortLabels = np.asarray(data2ShortLabels)
npdata2ShortLabels = np.delete(npdata2ShortLabels,
listToDelete,
axis=0)
# find left hand push and pull and label as pull and push respectively
data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels)
self.Y = dataVecReq
self.L = data2ShortLabels
# logging.info('\n'.join(data2Labels))
logging.info(self.Y.shape)
logging.info(len(self.L))
def convertToDict(self, rawData, mode, verbose):
data = dict()
firstPass = True
jointsList = []
objectsList = []
# logging.info('*******************')
# for j in self.paramsDict:
# logging.info(j, self.paramsDict[j]
# logging.info('*******************')
for t in rawData:
# parse skeleton data which has 9 sections by (x,y,z)
for i in range(self.numJoints):
a = i * 4
# if t[a] == 'shoulderCenter':
# t[a] = 'chest'
if firstPass:
data[t[a]] = [None]
data[t[a]] = (np.array(
[float(t[a + 1]),
float(t[a + 2]),
float(t[a + 3])]))
jointsList.append(t[a])
else:
arr = np.array(
[float(t[a + 1]),
float(t[a + 2]),
float(t[a + 3])])
if data[t[a]] is not None:
data[t[a]] = np.vstack((data[t[a]], arr))
else:
data[t[a]] = arr
currIdx = (self.numJoints * 4 - 1)
numObjs = (len(t) - currIdx) / 5
for i in range(numObjs):
a = currIdx + 1 + (i * 5)
if t[a] in data:
arr = np.array(
[float(t[a + 1]),
float(t[a + 2]),
float(t[a + 3])])
if data[t[a]] is not None:
data[t[a]] = np.vstack((data[t[a]], arr))
else:
data[t[a]] = arr
else:
data[t[a]] = [None]
data[t[a]] = np.array(
[float(t[a + 1]),
float(t[a + 2]),
float(t[a + 3])])
if mode == 'testing' or (mode != 'testing'
and t[a + 4] == '1'):
objectsList.append(t[a])
firstPass = False
if verbose:
logging.info('data has length = ' + str(len(data)) + ' joints')
logging.info('each joint has an array of shape ' +
str(data['head'].shape))
if self.paramsDict['filterData'] or 'vel' in self.paramsDict['components'] or \
'acc' in self.paramsDict['components']:
if verbose:
logging.info('Filtering data with hamming window of size ' +
str(self.paramsDict['filterWindow']))
for j in data.keys():
t1 = utils.smooth1D(data[j][:, 0],
self.paramsDict['filterWindow'])
t2 = utils.smooth1D(data[j][:, 1],
self.paramsDict['filterWindow'])
t3 = utils.smooth1D(data[j][:, 2],
self.paramsDict['filterWindow'])
data[j] = np.hstack([t1[:, None], t2[:, None], t3[:, None]])
if verbose:
logging.info('data has length = ' + str(len(data)) + ' joints')
logging.info('each joint has an array of shape ' +
str(data['head'].shape))
# convert data and number labels into windows.
# data is still in the form of a dictionary with the joints/objects as keys of the dict
# Text labels contained in labels
if verbose:
logging.info('')
noY = mode != 'testing'
if mode == 'testing':
offset = self.paramsDict['windowOffset']
else:
offset = 1
data2 = dict()
printExplanation = True
for num, key in enumerate(data):
data2[key] = None
xx, yy = utils.transformTimeSeriesToSeq(
data[key],
timeWindow=self.paramsDict['windowSize'],
offset=offset,
normalised=self.paramsDict['normaliseWindow'],
reduced=self.paramsDict['reduce'],
noY=noY)
if self.paramsDict['thresholdMovement'] or 'vel' in self.paramsDict['components'] or 'acc' in \
self.paramsDict['components']:
winSize = xx.shape[1] / 3
g = xx.size / winSize
xxshape1 = xx.shape[0]
xxshape2 = xx.shape[1]
flatxx = xx.flatten()
f = flatxx.reshape([g, winSize])
xx = f.reshape([xxshape1, xxshape2])
if self.paramsDict['thresholdMovement']:
if printExplanation and verbose:
logging.info('thresholding movement <' +
str(self.paramsDict['moveThresh']))
ranges = np.ptp(f, axis=1)
a = ranges < self.paramsDict['moveThresh']
b = ranges > -self.paramsDict['moveThresh']
res = list(np.where(np.logical_and(a, b))[0])
if self.paramsDict['normaliseWindow']:
f[res] = 0
else:
for ll in res:
f[ll] = f[ll][0]
if 'vel' in self.paramsDict['components']:
if printExplanation and verbose:
logging.info('Adding velocity to the feature vector')
xxvel = np.diff(f)
xxvel = xxvel.reshape([xxshape1, xxshape2 - 3])
xx = np.hstack([xx, xxvel])
if 'acc' in self.paramsDict['components']:
if printExplanation and verbose:
logging.info(
'Adding acceleration to the feature vector')
xxacc = np.diff(f, n=2)
xxacc = xxacc.reshape([xxshape1, xxshape2 - 6])
xx = np.hstack([xx, xxacc])
data2[key] = xx
printExplanation = False
if verbose:
logging.info('data has length = ' + str(len(data2)) + ' joints')
logging.info('each joint has an array of shape ' +
str(data2['head'].shape))
return data2, jointsList, objectsList
def readData(self, root_data_dir, participant_index, *args, **kw):
self.rawData, labelsList = self.diskDataToLiveData(root_data_dir)
data2, jointsList, objectsList = self.convertToDict(self.rawData, 'testing', verbose=self.verbose)
logging.info('unique labels' + str(set(labelsList)))
# extract a set of labels
labels = list(set(labelsList))
labels.sort()
logging.info('')
# convert text labels into numbers
labelNumsList = None
for n, k in enumerate(labelsList):
res = [m for m, l in enumerate(labels) if l == k]
if n == 0:
labelNumsList = np.array(res)
else:
labelNumsList = np.vstack([labelNumsList, res])
logging.info('shape of number labels:' +str(labelNumsList.shape))
uu, tmp = utils.transformTimeSeriesToSeq(labelNumsList, self.paramsDict['windowSize'],
self.paramsDict['windowOffset'], False, False)
data2NumLabels = uu
logging.info('windowed number labels shape:' + str(data2NumLabels.shape))
# now that labels are in windowed form it is time to
# assign them a text label again that describes them
# the rule here is if the window appertains to the same label,
# that label is assigned otherwise it is labelled as transition
data2Labels = []
for j in data2NumLabels:
numItems = list(set(j))
if len(numItems) == 1:
l = labels[int(numItems[0])]
data2Labels.append(l)
else:
# Another way to do this would be to label it according to 75% majority
# This would decrease the region size of the transition blocks
# which are currently dependant on windowSize
data2Labels.append('transition')
logging.info('after transition unique set ' + str(set(data2Labels)))
logging.info('windowed data labels compressed: ' + str(len(data2Labels)))
logging.info('')
# create list of specific joints to be used
jointsToUse = []
objectDict = dict()
handDict = dict()
for j in self.paramsDict['includeParts']:
if j == 'object':
for k in objectsList:
if k != 'partner':
objectDict[k] = (len(jointsToUse))
jointsToUse.append(k)
elif 'hand' in j:
handDict[j] = (len(jointsToUse))
jointsToUse.append(j)
else:
jointsToUse.append(j)
combineObjects = len(objectDict) > 1
combineHands = len(handDict) > 1
logging.info(jointsToUse)
logging.info(objectDict)
logging.info(handDict)
# concatenate data for all joints in a single vector
logging.info('')
dataVecAll = None
for j in jointsToUse:
if dataVecAll is None:
dataVecAll = data2[j]
else:
dataVecAll = np.hstack([dataVecAll, data2[j]])
itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse)
logging.info(dataVecAll.shape)
logging.info(itemsPerJoint)
self.itemsPerJoint = itemsPerJoint
logging.info('')
# ------------------------------------------------------------------
# it is now time to combine objects if multiple exist
#
logging.info('')
self.featureSequence = []
combinedObjs = dict()
if combineObjects and 'object' in self.paramsDict['includeParts']:
self.featureSequence.append('object')
logging.info('Combining Objects')
for n in objectDict:
idxBase = objectDict[n] * itemsPerJoint
combinedObjs[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
logging.info(combinedObjs[n].shape)
logging.info(dataVecAll.shape)
logging.info('')
# it is now time to combine hands if multiple exist
combinedHands = dict()
if combineHands and self.paramsDict['combineHands'] and \
len([s for s in self.paramsDict['includeParts'] if 'hand' in s]) > 0:
logging.info('Combining hands')
self.handsCombined = True
self.featureSequence.append('hand')
for n in handDict:
idxBase = handDict[n] * itemsPerJoint
combinedHands[n] = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
logging.info(combinedHands[n].shape)
logging.info(dataVecAll.shape)
else:
self.handsCombined = False
logging.info(jointsToUse)
otherJoints = None
for j, item in enumerate(jointsToUse):
if self.handsCombined:
if item not in handDict and item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if otherJoints is None:
otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
else:
otherJoints = np.hstack([otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint]])
else:
if item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if otherJoints is None:
otherJoints = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
else:
otherJoints = np.hstack([otherJoints, dataVecAll[:, idxBase:idxBase + itemsPerJoint]])
if otherJoints is not None:
logging.info(otherJoints.shape)
self.listOfVectorsToClassify = []
for j in self.featureSequence:
if j == 'object':
for k in objectsList:
if k != 'partner':
self.listOfVectorsToClassify.append([k])
elif 'hand' in j:
if self.handsCombined:
a = copy.deepcopy(self.listOfVectorsToClassify)
b = copy.deepcopy(self.listOfVectorsToClassify)
if len(self.listOfVectorsToClassify) > 0:
for l, m in enumerate(self.listOfVectorsToClassify):
a[l].append('handLeft')
b[l].append('handRight')
self.listOfVectorsToClassify = a + b
else:
self.listOfVectorsToClassify.append(['handLeft'])
self.listOfVectorsToClassify.append(['handRight'])
else:
for l, m in enumerate(self.listOfVectorsToClassify):
self.listOfVectorsToClassify[l].append(j)
else:
for l, m in enumerate(self.listOfVectorsToClassify):
self.listOfVectorsToClassify[l].append(j)
logging.info('Vectors to Classify:')
for j in self.listOfVectorsToClassify:
logging.info("\t" + str(j))
dataVecReq = None
objSection = None
if combinedObjs:
objSection = None
for j in self.listOfVectorsToClassify:
logging.info(str(j[0]))
if objSection is None:
objSection = combinedObjs[j[0]]
else:
objSection = np.vstack([objSection, combinedObjs[j[0]]])
dataVecReq = objSection
logging.info(str(objSection.shape))
handsSection = None
if combinedHands:
for j in self.listOfVectorsToClassify:
for l in j:
if 'hand' in l:
if handsSection is None:
handsSection = combinedHands[l]
else:
handsSection = np.vstack([handsSection, combinedHands[l]])
if dataVecReq is None:
dataVecReq = handsSection
else:
dataVecReq = np.hstack([dataVecReq, handsSection])
logging.info(str(handsSection.shape))
othersSection = None
if otherJoints is not None:
for j in self.listOfVectorsToClassify:
logging.info(str(j[:]))
if othersSection is None:
othersSection = otherJoints
else:
othersSection = np.vstack([othersSection, otherJoints])
if dataVecReq is None:
dataVecReq = othersSection
else:
dataVecReq = np.hstack([dataVecReq, othersSection])
logging.info(str(dataVecReq.shape))
del handsSection, othersSection, objSection, combinedHands, combinedObjs, otherJoints
# Also augment the labels list
data2LabelsAugment = []
for j in self.listOfVectorsToClassify:
data2LabelsAugment.append([])
for j in data2Labels:
splitLabel = j.split('_')
action = '_'.join(splitLabel[:2])
if len(splitLabel) > 2:
obj = splitLabel[2]
hand = splitLabel[4]
if combineHands:
handSubList = [k for k in self.listOfVectorsToClassify if 'hand' + hand.capitalize() in k]
if combineObjects:
vec = [f for f in handSubList if obj in f][0]
else:
vec = handSubList[0]
else:
vec = [f for f in self.listOfVectorsToClassify if obj in f][0]
# logging.info(data2Labels.index(j), vec)
# printStr = ''
for n, k in enumerate(self.listOfVectorsToClassify):
if vec == k:
data2LabelsAugment[n].append(action)
# printStr += action + '\t'
# else:
data2LabelsAugment[n].append('idle')
# printStr += '\tidle'
# logging.info(data2LabelsAugment[n][-1],)
# print
else:
obj = ''
hand = ''
printStr = ''
for n, k in enumerate(self.listOfVectorsToClassify):
# printStr += action + '\t'
data2LabelsAugment[n].append(action)
# logging.info(data2LabelsAugment[n][-1],)
# print
# logging.info(action, obj, hand)
# logging.info('---------------------')
# logging.info('before augment', set(data2Labels))
data2Labels = []
for j in data2LabelsAugment:
data2Labels += j
# logging.info('after augment', set(data2Labels)
logging.info('labels ' + str(len(data2Labels)))
logging.info('data ' + str(dataVecReq.shape))
self.allDataDict = dict()
self.allDataDict['Y'] = copy.deepcopy(dataVecReq)
self.allDataDict['L'] = copy.deepcopy(data2Labels)
# ---------------------------------------------------------------------------------
data2ShortLabels = []
for j in data2Labels:
splitLabel = j.split('_')
slabel = ('_'.join(splitLabel[:2]))
if splitLabel[0] == 'push' or splitLabel[0] == 'pull':
if splitLabel[-1] == 'no':
add = splitLabel[-2]
else:
add = splitLabel[-1]
if add == 'left' and self.paramsDict['flip']:
if splitLabel[0] == 'push':
splitLabel[0] = 'pull'
else:
splitLabel[0] = 'push'
slabel = ('_'.join(splitLabel[:2]))
if self.paramsDict['sepRL']:
slabel += '_' + add
data2ShortLabels.append(slabel)
self.data2Labels = copy.deepcopy(data2ShortLabels)
logging.info('shortLabels len ' + str(set(self.data2Labels)))
if self.paramsDict['sepRL']:
if 'pull_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index('pull_object') == 'pull_object_right'
self.paramsDict['actionsAllowedList'].append('pull_object_left')
if 'push_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index('push_object') == 'push_object_right'
self.paramsDict['actionsAllowedList'].append('push_object_left')
# remove labels which will not be trained
logging.info('actions allowed: ' + str(self.paramsDict['actionsAllowedList']))
listToDelete = []
for n in reversed(range(len(data2Labels))):
if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \
'no' in data2Labels[n]:
listToDelete.append(n)
dataVecReq = np.delete(dataVecReq, listToDelete, axis=0)
npdata2ShortLabels = np.asarray(data2ShortLabels)
npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0)
# find left hand push and pull and label as pull and push respectively
data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels)
self.Y = dataVecReq
self.L = data2ShortLabels
# logging.info('\n'.join(data2Labels))
logging.info(self.Y.shape)
logging.info(len(self.L))
def readData1(self, root_data_dir, participant_index, *args, **kw):
self.rawData, labelsList = self.diskDataToLiveData(root_data_dir)
data2, jointsList, objectsList = self.convertToDict(self.rawData, 'testing', verbose=self.verbose)
# extract a set of labels
labels = list(set(labelsList))
labels.sort()
logging.info('')
# convert text labels into numbers
labelNumsList = None
for n, k in enumerate(labelsList):
res = [m for m, l in enumerate(labels) if l == k]
if n == 0:
labelNumsList = np.array(res)
else:
labelNumsList = np.vstack([labelNumsList, res])
logging.info('shape of number labels:' + str(labelNumsList.shape))
uu, tmp = utils.transformTimeSeriesToSeq(labelNumsList, self.paramsDict['windowSize'],
self.paramsDict['windowOffset'], False, False)
data2NumLabels = uu
logging.info('windowed number labels shape:' + str(data2NumLabels.shape))
# now that labels are in windowed form it is time to
# assign them a text label again that describes them
# the rule here is if the window appertains to the same label,
# that label is assigned otherwise it is labelled as transition
data2Labels = []
for j in data2NumLabels:
numItems = list(set(j))
if len(numItems) == 1:
l = labels[int(numItems[0])]
data2Labels.append(l)
else:
# Another way to do this would be to label it according to 75% majority
# This would decrease the region size of the transition blocks
# which are currently dependant on windowSize
data2Labels.append('transition')
logging.info('windowed data labels compressed:' + str(len(data2Labels)))
logging.info('')
# create list of specific joints to be used
jointsToUse = []
objectDict = dict()
handDict = dict()
for j in self.paramsDict['includeParts']:
if j == 'object':
for k in objectsList:
if k != 'partner':
objectDict[k] = (len(jointsToUse))
jointsToUse.append(k)
elif 'hand' in j:
handDict[j] = (len(jointsToUse))
jointsToUse.append(j)
else:
jointsToUse.append(j)
combineObjects = len(objectDict) > 1
combineHands = len(handDict) > 1
logging.info(jointsToUse)
logging.info(objectDict)
logging.info(handDict)
# concatenate data for all joints in a single vector
logging.info('')
dataVecAll = None
for j in jointsToUse:
if dataVecAll is None:
dataVecAll = data2[j]
else:
dataVecAll = np.hstack([dataVecAll, data2[j]])
itemsPerJoint = dataVecAll.shape[1] / len(jointsToUse)
logging.info(dataVecAll.shape)
logging.info(itemsPerJoint)
self.itemsPerJoint = itemsPerJoint
logging.info('')
# it is now time to combine objects if multiple exist
#
self.featureSequence = ['object']
logging.info('')
combinedObjs = None
if combineObjects:
logging.info('Combining Objects')
for j in range(len(data2Labels)):
# logging.info(data2Labels[j])
if len(data2Labels[j].split('_')) > 2:
idxBase = objectDict[data2Labels[j].split('_')[2]] * itemsPerJoint
else:
idxBase = objectDict[objectDict.keys()[0]] * itemsPerJoint
if combinedObjs is None:
combinedObjs = dataVecAll[j, idxBase:idxBase + itemsPerJoint]
else:
combinedObjs = np.vstack([combinedObjs, dataVecAll[j, idxBase:idxBase + itemsPerJoint]])
logging.info(combinedObjs.shape)
logging.info(dataVecAll.shape)
logging.info('')
# it is now time to combine hands if multiple exist
combinedHands = None
if combineHands and self.paramsDict['combineHands']:
logging.info('Combining hands')
self.handsCombined = True
self.featureSequence.append('hand')
for j in range(len(data2Labels)):
if len(data2Labels[j].split('_')) > 2:
idxBase = handDict[data2Labels[j].split('_')[3] + data2Labels[j].split('_')[4].capitalize()] * \
itemsPerJoint
else:
idxBase = handDict[handDict.keys()[0]] * itemsPerJoint
if combinedHands is None:
combinedHands = dataVecAll[j, idxBase:idxBase + itemsPerJoint]
else:
combinedHands = np.vstack([combinedHands, dataVecAll[j, idxBase:idxBase + itemsPerJoint]])
logging.info(dataVecAll.shape)
logging.info(combinedHands.shape)
else:
self.handsCombined = False
dataVecReq = None
if combinedHands is not None:
dataVecReq = combinedHands
if combinedObjs is not None:
if dataVecReq is None:
dataVecReq = combinedObjs
else:
dataVecReq = np.hstack([dataVecReq, combinedObjs])
logging.info(jointsToUse)
for j, item in enumerate(jointsToUse):
if self.handsCombined:
if item not in handDict and item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if dataVecReq is None:
dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
else:
dataVecReq = np.hstack([dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint]])
else:
if item not in objectDict:
self.featureSequence.append(item)
idxBase = j * itemsPerJoint
if dataVecReq is None:
dataVecReq = dataVecAll[:, idxBase:idxBase + itemsPerJoint]
else:
dataVecReq = np.hstack([dataVecReq, dataVecAll[:, idxBase:idxBase + itemsPerJoint]])
logging.info(dataVecReq.shape)
logging.info(len(data2Labels))
logging.info('')
self.dataVec = copy.deepcopy(dataVecReq)
data2ShortLabels = []
for j in data2Labels:
splitLabel = j.split('_')
slabel = ('_'.join(splitLabel[:2]))
if splitLabel[0] == 'push' or splitLabel[0] == 'pull':
if splitLabel[-1] == 'no':
add = splitLabel[-2]
else:
add = splitLabel[-1]
if add == 'left' and self.paramsDict['flip']:
if splitLabel[0] == 'push':
splitLabel[0] = 'pull'
else:
splitLabel[0] = 'push'
slabel = ('_'.join(splitLabel[:2]))
if self.paramsDict['sepRL']:
slabel += '_' + add
data2ShortLabels.append(slabel)
self.data2Labels = copy.deepcopy(data2ShortLabels)
if self.paramsDict['sepRL']:
if 'pull_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index('pull_object') == 'pull_object_right'
self.paramsDict['actionsAllowedList'].append('pull_object_left')
if 'push_object' in self.paramsDict['actionsAllowedList']:
self.paramsDict['actionsAllowedList'].index('push_object') == 'push_object_right'
self.paramsDict['actionsAllowedList'].append('push_object_left')
# remove labels which will not be trained
listToDelete = []
for n in reversed(range(len(data2Labels))):
if len([j for j in self.paramsDict['actionsAllowedList'] if j in data2Labels[n]]) == 0 or \
'no' in data2Labels[n]:
listToDelete.append(n)
dataVecReq = np.delete(dataVecReq, listToDelete, axis=0)
npdata2ShortLabels = np.asarray(data2ShortLabels)
npdata2ShortLabels = np.delete(npdata2ShortLabels, listToDelete, axis=0)
# find left hand push and pull and label as pull and push respectively
data2ShortLabels = np.ndarray.tolist(npdata2ShortLabels)
self.Y = dataVecReq
self.L = data2ShortLabels
# logging.info('\n'.join(data2Labels))
logging.info(self.Y.shape)
logging.info(len(self.L))
# now that all joints are in the form of a window, time to create
# all possible vectors to classify
self.allDataDict = dict()
self.allDataDict['Y'] = self.dataVec
self.allDataDict['L'] = self.data2Labels
listOfVectorsToClassify = self.listOfClassificationVectors(self.featureSequence, objectsList)
for j in listOfVectorsToClassify:
logging.info(j)
def convertToDict(self, rawData, mode, verbose):
data = dict()
firstPass = True
jointsList = []
objectsList = []
# logging.info('*******************')
# for j in self.paramsDict:
# logging.info(j, self.paramsDict[j]
# logging.info('*******************')
for t in rawData:
# parse skeleton data which has 9 sections by (x,y,z)
for i in range(self.numJoints):
a = i * 4
# if t[a] == 'shoulderCenter':
# t[a] = 'chest'
if firstPass:
data[t[a]] = [None]
data[t[a]] = (np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])]))
jointsList.append(t[a])
else:
arr = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])])
if data[t[a]] is not None:
data[t[a]] = np.vstack((data[t[a]], arr))
else:
data[t[a]] = arr
currIdx = (self.numJoints * 4 - 1)
numObjs = (len(t) - currIdx) / 5
for i in range(numObjs):
a = currIdx + 1 + (i * 5)
if t[a] in data:
arr = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])])
if data[t[a]] is not None:
data[t[a]] = np.vstack((data[t[a]], arr))
else:
data[t[a]] = arr
else:
data[t[a]] = [None]
data[t[a]] = np.array([float(t[a + 1]), float(t[a + 2]), float(t[a + 3])])
if mode == 'testing' or (mode != 'testing' and t[a+4] == '1'):
objectsList.append(t[a])
firstPass = False
if verbose:
logging.info('data has length = ' + str(len(data)) + ' joints')
logging.info('each joint has an array of shape ' + str(data['head'].shape))
if self.paramsDict['filterData'] or 'vel' in self.paramsDict['components'] or \
'acc' in self.paramsDict['components']:
if verbose:
logging.info('Filtering data with hamming window of size ' + str(self.paramsDict['filterWindow']))
for j in data.keys():
t1 = utils.smooth1D(data[j][:, 0], self.paramsDict['filterWindow'])
t2 = utils.smooth1D(data[j][:, 1], self.paramsDict['filterWindow'])
t3 = utils.smooth1D(data[j][:, 2], self.paramsDict['filterWindow'])
data[j] = np.hstack([t1[:, None], t2[:, None], t3[:, None]])
if verbose:
logging.info('data has length = ' + str(len(data)) + ' joints')
logging.info('each joint has an array of shape ' + str(data['head'].shape))
# convert data and number labels into windows.
# data is still in the form of a dictionary with the joints/objects as keys of the dict
# Text labels contained in labels
if verbose:
logging.info('')
noY = mode != 'testing'
if mode == 'testing':
offset = self.paramsDict['windowOffset']
else:
offset = 1
data2 = dict()
printExplanation = True
for num, key in enumerate(data):
data2[key] = None
xx, yy = utils.transformTimeSeriesToSeq(data[key], timeWindow=self.paramsDict['windowSize'],
offset=offset,
normalised=self.paramsDict['normaliseWindow'],
reduced=self.paramsDict['reduce'], noY=noY)
if self.paramsDict['thresholdMovement'] or 'vel' in self.paramsDict['components'] or 'acc' in \
self.paramsDict['components']:
winSize = xx.shape[1] / 3
g = xx.size / winSize
xxshape1 = xx.shape[0]
xxshape2 = xx.shape[1]
flatxx = xx.flatten()
f = flatxx.reshape([g, winSize])
xx = f.reshape([xxshape1, xxshape2])
if self.paramsDict['thresholdMovement']:
if printExplanation and verbose:
logging.info('thresholding movement <' + str(self.paramsDict['moveThresh']))
ranges = np.ptp(f, axis=1)
a = ranges < self.paramsDict['moveThresh']
b = ranges > -self.paramsDict['moveThresh']
res = list(np.where(np.logical_and(a, b))[0])
if self.paramsDict['normaliseWindow']:
f[res] = 0
else:
for ll in res:
f[ll] = f[ll][0]
if 'vel' in self.paramsDict['components']:
if printExplanation and verbose:
logging.info('Adding velocity to the feature vector')
xxvel = np.diff(f)
xxvel = xxvel.reshape([xxshape1, xxshape2 - 3])
xx = np.hstack([xx, xxvel])
if 'acc' in self.paramsDict['components']:
if printExplanation and verbose:
logging.info('Adding acceleration to the feature vector')
xxacc = np.diff(f, n=2)
xxacc = xxacc.reshape([xxshape1, xxshape2 - 6])
xx = np.hstack([xx, xxacc])
data2[key] = xx
printExplanation = False
if verbose:
logging.info('data has length = ' + str(len(data2)) + ' joints')
logging.info('each joint has an array of shape ' + str(data2['head'].shape))
return data2, jointsList, objectsList
# #----------- Train a standard model
# m1=SAM.SAM_Core.LFM()
# m1.store(observed={'Y':Y1[0:Ntr,:]}, inputs=X1[0:Ntr,:], Q=None, kernel=None, num_inducing=num_inducing)
# m1.learn()
# ret = m1.visualise()
# y_pred_standard = m1.pattern_completion(X1[Ntr:,:])[0]
# pb.figure()
# pb.plot(X1[Ntr:,:],Y1[Ntr:,:], 'x-')
# pb.plot(X1[Ntr:,:],y_pred_standard, 'ro-')
# pb.legend(('True','Pred'))
# pb.title('Standard GP')
# # ---------------------------------------
# Create transformed data (autoregressive dataset)
ws = 10 # Windowsize
xx, yy = autoregressive.transformTimeSeriesToSeq(Y1, ws)
# uu,tmp = transformTimeSeriesToSeq(U1, ws)
# Test the above: np.sin(uu) - xx
# uu = yy**2 -2*yy + 5 + np.random.randn(*yy.shape) * 0.005
U1 = Y1**2 - 2 * Y1 + 5 + np.random.randn(*Y1.shape) * 0.005
uu, tmp = autoregressive.transformTimeSeriesToSeq(U1, ws)
Xtr = xx[0:Ntr, :]
Xts = xx[Ntr:, :]
Ytr = yy[0:Ntr, :]
Yts = yy[Ntr:, :]
Utr = uu[0:Ntr, :]
Uts = uu[Ntr:, :]