def main():
path='/local/attale00/'
allFiles = utils.getAllFiles(path+'Multi-PIE/extracted')
allLabelFiles = utils.getAllFiles(path+'Multi-PIE/db_labels')
np.random.shuffle(allLabelFiles)
#get the labels from the database, for each person
sliceat=250
labelstest=utils.parseLabelFiles(path+'Multi-PIE/db_labels','sex',allLabelFiles[0:sliceat],cutoffSeq='',suffix='')
labelstraining = utils.parseLabelFiles(path+'Multi-PIE/db_labels','sex',allLabelFiles[sliceat:],cutoffSeq='',suffix='')
#now generate the label dict for each file
labsTest={}
labsTraining={}
for f in allFiles:
if labelstest.has_key(f[0:3]+'.labels'):
labsTest[f]=labelstest[f[0:3]+'.labels']
elif labelstraining.has_key(f[0:3]+'.labels'):
labsTraining[f]=labelstraining[f[0:3]+'.labels' ]
testSet = fg.dataContainer(labsTest)
trainingSet = fg.dataContainer(labsTraining)
roi=(0,64,0,64)
ppc=(8,8)
cpb=(8,8)
fg.getHogFeature(testSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None,pixels_per_cell=ppc,cells_per_block=cpb)
fg.getHogFeature(trainingSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None,pixels_per_cell=ppc, cells_per_block=cpb)
testSet.targetNum=map(utils.mapSexLabel2Two,testSet.target)
trainingSet.targetNum = map(utils.mapSexLabel2Two,trainingSet.target)
rf1=classifierUtils.standardRF(max_features=np.sqrt(len(testSet.data[0])))
rf2=classifierUtils.standardRF(max_features=np.sqrt(len(trainingSet.data[0])))
rf1.fit(testSet.data,testSet.targetNum)
s=rf1.score(trainingSet.data,trainingSet.targetNum)
trainingSet.classifiedAs=rf1.predict(trainingSet.data)
trainingSet.hasBeenClassified=True
classifierUtils.evaluateClassification(trainingSet,{0:'male',1:'female'})
print 'Score: {}'.format(s)
print '----------other way around ----\n'
rf2.fit(trainingSet.data,trainingSet.targetNum)
s=rf2.score(testSet.data,testSet.targetNum)
testSet.classifiedAs=rf2.predict(testSet.data)
testSet.hasBeenClassified=True
classifierUtils.evaluateClassification(testSet,{0:'male',1:'female'})
print 'Score: {}'.format(s)
def plotExcerpt(path='/local/attale00/GoodPose/extracted_alpha/grayScale',roi=(0,250,60,452),showDiff=''):
files = utils.getAllFiles(path)
im = cv2.imread(path+'/'+files[0],-1)
sh=np.shape(im)
cv2.namedWindow('Panorama')
big = np.zeros((sh[0],sh[1]*2),dtype='uint8')
for f in files:
im = cv2.imread(path+'/'+f,-1)
try:
im = cv2.cvtColor(im,cv2.cv.CV_RGB2GRAY)
except Exception as e:
pass
big[:,0:sh[1]]=im
cv2.rectangle(big,(roi[2],roi[1]),(roi[3],roi[0]),255,2)
excerpt = im[roi[0]:roi[1],roi[2]:roi[3]]
if showDiff in ['y','Y']:
excerpt = np.diff(excerpt,n=1,axis=0)
if showDiff in ['X','x']:
excerpt = np.diff(excerpt,n=1,axis=1)
big[roi[0]:roi[1],sh[1]+roi[2]:sh[1]+roi[3]]=excerpt
cv2.imshow('Panorama',big)
k=cv2.waitKey(0)
if k==27:
break
cv2.destroyAllWindows()
print('done')
def main(mode):
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
fg_mode = 0
size=(4,12)
overlap=2
#size=(40,120)
fg.getImagePatchStat(testSet,path=path_ea,patchSize=size,overlap = overlap,mode=fg_mode)
print 'feature vector length: {}'.format(len(testSet.data[0]))
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=13,max_depth=40)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
rValues = classifierUtils.dissectedCV(rf,testSet)
pickle.dump(rValues,open('errorpatch_size_{}'.format(size[0]),'w'))
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf)
else:
print 'not doing anything'
def errorPatch():
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/mouth_img_error_multiPie/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
#
fileNames = labeledImages;
testSet = fg.dataContainer(labs)
fg.getImagePatchStat(testSet,path=path_ea,patchSize=(4,12),overlap = 2)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
clfPath = '/home/attale00/Desktop/classifiers/errorpatches/rferror'
f=file(clfPath,'r')
print 'classifier used: '+ f.name
clf = pickle.load(f)
testSet.classifiedAs=clf.predict(testSet.data)
testSet.probabilities=clf.predict_proba(testSet.data)
return testSet
def main(mode):
path = "/local/attale00/extracted_pascal__4__Multi-PIE"
path_ea = path + "/color128/"
allLabelFiles = utils.getAllFiles("/local/attale00/a_labels")
labeledImages = [i[0:16] + ".png" for i in allLabelFiles]
# labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
labs = utils.parseLabelFiles(
"/local/attale00/a_labels", "mouth", labeledImages, cutoffSeq=".png", suffix="_face0.labels"
)
testSet = fg.dataContainer(labs)
roi = (50, 74, 96, 160)
X = fg.getAllImagesFlat(path_ea, testSet.fileNames, (128, 256), roi=roi)
# perform ICA
if mode not in ["s", "v"]:
ica = FastICA(n_components=100, whiten=True)
ica.fit(X)
meanI = np.mean(X, axis=0)
X1 = X - meanI
data = ica.transform(X1)
filters = ica.components_
elif mode in ["s", "v"]:
W = np.load("/home/attale00/Desktop/classifiers/ica/filter1.npy")
m = np.load("/home/attale00/Desktop/classifiers/ica/meanI1.npy")
X1 = X - m
data = np.dot(X1, W.T)
for i in range(len(testSet.data)):
testSet.data[i].extend(data[i, :])
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
# fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
# fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)
testSet.targetNum = map(utils.mapMouthLabels2Two, testSet.target)
rf = classifierUtils.standardRF(max_features=np.sqrt(len(testSet.data[0])), min_split=5, max_depth=40)
if mode in ["s", "v"]:
print "Classifying with loaded classifier"
classifierUtils.classifyWithOld(
path, testSet, mode, clfPath="/home/attale00/Desktop/classifiers/ica/rf128ICA_1"
)
elif mode in ["c"]:
print "cross validation of data"
print "Scores"
# print classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
# _cvDissect(testSet,rf)
classifierUtils.dissectedCV(rf, testSet)
print "----"
elif mode in ["save"]:
print "saving new classifier"
_saveRF(testSet)
else:
print "not doing anything"
def main(mode):
path = '/local/attale00/extracted_pascal__4__Multi-PIE'
path_ea = path+'/color128/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
#labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
#fileNames = utils.getAllFiles(path_ea);
#labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(50,74,96,160)
#roi=(44,84,88,168)
# eM=np.load('/home/attale00/Desktop/mouthMask.npy')
# m=cv2.resize(np.uint8(eM),(256,256));
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# dil = cv2.dilate(m,strel)
#
# m=dil>0;
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 8, cells_per_block=(6,2),maskFromAlpha=False)
fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,'s')
elif mode in ['c']:
print 'cross validation of data'
print 'Scores'
#print classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
#_cvDissect(testSet,rf)
classifierUtils.dissectedCV(rf,testSet)
print '----'
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet)
else:
print 'not doing anything'
def main(mode):
path='/local/attale00/'
allFiles = utils.getAllFiles(path+'Multi-PIE/extracted')
allLabelFiles = utils.getAllFiles(path+'Multi-PIE/labels')
#allLabelFiles = utils.getAllFiles(path+'a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
#labs=utils.parseLabelFiles(path+'a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi = (0,32,0,64)
#roi = (128,256,0,256)
eM=np.load('/home/attale00/Desktop/mouthMask.npy')
m=cv2.resize(np.uint8(eM),(256,256));
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
dil = cv2.dilate(m,strel)
m=dil>0;
fg.getHogFeature(testSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None)
fg.getColorHistogram(testSet,(50,190,110,402),path = path+'/Multi-PIE/extracted/',ending=None,colorspace='lab',range=(1.,255.0),bins = 20)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
#testSet.targetNum = map(utils.mapGlassesLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
classifierUtils.dissectedCV(rf,testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf)
else:
print 'not doing anything'
def run_individual_parameters(i,
data_path,
extension,
save_folder,
GRP=False,
compute_p=True,
**preprocessing_params):
"""
Arguments:
GRP -- True if processing the GRP data
"""
files = utils.getAllFiles(data_path, extension)
if GRP:
file = files[0]
filename = '{:02}'.format(
i) # Save the results by the subjects number
subject_id = i
else:
file = files[i]
filename = utils.basename(file)
subject_id = None
os.makedirs("Results/{}/AM/idx".format(save_folder), exist_ok=True)
os.makedirs("Results/{}/AM/p_values".format(save_folder),
exist_ok=True)
print("Processing", i, ":", filename)
if os.path.exists('Results/{}/AM/p_values/{}.csv'.format(
save_folder, filename)):
exit()
param_file = 'Results/{}/AIS/idx/{}.csv'.format(save_folder, filename)
df, param_df = utils.loadData(file,
get_params=True,
param_file=param_file,
subject_id=subject_id)
data = utils.preprocess(df, **preprocessing_params)
results, p_values = getLocalsForAllRegions(data,
param_df,
compute_p=compute_p)
# Add back the trimmed sections
padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
preprocessing_params.get('trim_end', 0)))
results = np.pad(results, padding, mode='constant', constant_values=0)
pd.DataFrame(results).to_csv('Results/{}/AM/{}_AM.csv'.format(
save_folder, filename),
index=None,
header=None)
pd.DataFrame(p_values).to_csv('Results/{}/AM/p_values/{}.csv'.format(
save_folder, filename),
index=None,
header=None)
try:
utils.plotHeatmap(results, divergent=True)
except:
pass
def patches():
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/multiPIE_cropped3/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
fileNames = labeledImages;
testSet = fg.dataContainer(labs)
roi=(0,37,0,115)
roi=None
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
W=np.load('/home/attale00/Desktop/classifiers/patches/filter2.npy')
m=np.load('/home/attale00/Desktop/classifiers/patches/meanI2.npy')
X1=X-m
data=np.dot(X1,W.T)
for i in range(len(fileNames)):
testSet.data[i].extend(data[i,:])
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, pixels_per_cell=(24,8),cells_per_block=(3,3),maskFromAlpha=False)
fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
clfPath = '/home/attale00/Desktop/classifiers/patches/rfICAHogColor'
f=file(clfPath,'r')
print 'classifier used: '+ f.name
clf = pickle.load(f)
testSet.classifiedAs=clf.predict(testSet.data)
testSet.probabilities=clf.predict_proba(testSet.data)
return testSet
def main(mode):
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
components = 150
roi=None
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
# X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(120,40),roi=roi,resizeFactor = .5)
#
# perform ICA
if mode not in ['s','v']:
ica = FastICA(n_components=components,whiten=True)
ica.fit(X)
meanI=np.mean(X,axis=0)
X1=X-meanI
data=ica.transform(X1)
filters=ica.components_
elif mode in ['s','v']:
W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
X1=X-m
data=np.dot(X1,W.T)
for i in range(len(fileNames)):
testSet.data[i].extend(data[i,:])
print 'feature vector length: {}'.format(len(testSet.data[0]))
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=13,max_depth=40)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
rValues = classifierUtils.dissectedCV(rf,testSet)
pickle.dump(rValues,open('errorpatch_ica','w'))
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf,filters=filters,meanI=meanI)
else:
print 'not doing anything'
def main(nJobs = 1):
path = '/local/attale00/GoodPose/extracted_alpha/grayScale64'
fileNames = utils.getAllFiles(path);
labs=utils.parseLabelFiles('/local/attale00/GoodPose'+'/mouth_labels','mouth',fileNames,cutoffSeq='_0.png',suffix='_face0.labels')
print('-----computing Features-----')
roi2 = (0,32,0,64)
mouthSet = fg.dataContainer(labs)
#load the mask for the mouth room pixels and dilate it
eM=np.load('/home/attale00/Desktop/mouthMask.npy')
m=cv2.resize(np.uint8(eM),(256,256));
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
dil = cv2.dilate(m,strel)
m=dil>0;
#get the features
fg.getHogFeature(mouthSet,roi2,path=path+'/',ending=None,extraMask = None)
#map the string labels to numbers (required by sklearn)
#change the mapping here for different classifiers
mouthSet.targetNum=map(utils.mapMouthLabels2Two,mouthSet.target)
n_estimators = 100
min_split = 10
max_depth = 20
max_features = np.sqrt(len(mouthSet.data[0]))
rf = classifierUtils.standardRF(max_features = max_features)
rf2=classifierUtils.standardRF(max_features=max_features)
score=classifierUtils.standardCrossvalidation(rf2,mouthSet)
rf.fit(mouthSet.data,mouthSet.targetNum)
pickle.dump(rf,open('/home/attale00/Desktop/classifiers/RandomForestMouthclassifier_12','w'))
f=open('/home/attale00/Desktop/classifiers/RandomForestMouthclassifier_12.txt','w')
f.write('Trained on aflw\n')
f.write('Attribute: mouth' )
f.write('Features: getHogFeature(mouthSet,roi2,path=path,ending=None,extraMask = m) on 64*64 grayScale 3 direction bins \n')
f.write('ROI:(0,32,0,64)\n')
f.write('labels: closed, narrow: 0, open, wideOpen: 1\n')
f.write('CV Score: {}\n'.format(score))
f.close()
def aflwPose():
path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#
fileNames = utils.getAllFiles(path_ea);
path_pose = '/local/attale00/poseLabels/aflw'
poseLabelaflw = utils.parseLabelFiles(path_pose,'rotY',fileNames,cutoffSeq='.png',suffix='.frog')
poses_aflw = [np.double(i)*180/np.pi for i in poseLabelaflw.values()]
return np.array(poses_aflw)
def main(mode):
path = '/local/attale00/AFLW_ALL'
path_ea = path+'/color256/'
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(88,165,150,362)
#roi=(44,84,88,168)
# eM=np.load('/home/attale00/Desktop/mouthMask.npy')
# m=cv2.resize(np.uint8(eM),(256,256));
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# dil = cv2.dilate(m,strel)
#
# m=dil>0;
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(26,9),maskFromAlpha=False)
fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
print len(testSet.data)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
#classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
classifierUtils.dissectedCV(rf,testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet)
else:
print 'not doing anything'
def multiPiePose():
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
path_pose = '/local/attale00/poseLabels/multipie'
poseLabel = utils.parseLabelFiles(path_pose,'rotY',labeledImages,cutoffSeq='.png',suffix='.frog')
poses = np.array([np.double(i)*180/np.pi for i in poseLabel.values()])
return poses
def plotThreshold(path='/local/attale00/GoodPose/extracted_alpha/grayScale',thresh=100):
files = utils.getAllFiles(path)
big = np.zeros((512,2*512),dtype='uint8')
for f in files:
im = cv2.imread(path+'/'+f,-1)
big[:,0:512]=im
ret,imbw = cv2.threshold(im,thresh,255,cv2.THRESH_BINARY)
big[:,512:1024]=imbw
cv2.imshow('test',big)
k=cv2.waitKey(0)
if k==27:
return
print('done')
def main(mode):
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
roi=None
testSet = fg.dataContainer(labs)
testSetMirror = fg.dataContainer(labs)
for f in range(len(testSetMirror.fileNames)):
testSetMirror.fileNames[f]+='M'
orientations = 9
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = orientations, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)
fg.getHogFeature(testSetMirror,roi,path='/local/attale00/AFLW_cropped/mirrored/', ending='.png',orientations = orientations, cells_per_block=(3,3),pixels_per_cell=(24,8))
testSet.addContainer(testSetMirror)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = 30,min_split=12,max_depth=70)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
rValues = classifierUtils.dissectedCV(rf,testSet)
pickle.dump(rValues,open('patches_cv_hog_{}'.format(orientations),'w'))
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf)
else:
print 'not doing anything'
def plotExcerpt(path='/local/attale00/GoodPose/extracted_alpha/grayScale',roi=(0,250,60,452)):
files = utils.getAllFiles(path)
cv2.namedWindow('thresh')
big = np.zeros((512,2*512),dtype='uint8')
for f in files:
im = cv2.imread(path+'/'+f,-1)
im = cv2.cvtColor(im,cv2.cv.CV_RGB2GRAY)
big[:,0:512]=im
big[roi[0]:roi[1],512:512+roi[3]-roi[2]]=im[roi[0]:roi[1],roi[2]:roi[3]]
cv2.imshow('test',big)
k=cv2.waitKey(0)
if k==27:
return
print('done')
def main(mode):
path = '/local/attale00/extracted_pascal__4__Multi-PIE'
path_ea = path+'/color256/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
#labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(88,165,150,362)
#roi=(44,84,88,168)
# eM=np.load('/home/attale00/Desktop/mouthMask.npy')
# m=cv2.resize(np.uint8(eM),(256,256));
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# dil = cv2.dilate(m,strel)
#
# m=dil>0;
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(26,9),maskFromAlpha=False)
fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,'v')
elif mode in ['c']:
print 'cross validation of data'
_cross_validate(testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet)
else:
print 'not doing anything'
def texture():
path = '/local/attale00/extracted_pascal__4__Multi-PIE'
path_ea = path+'/color128/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(50,74,96,160)
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)
W=np.load('/home/attale00/Desktop/classifiers/ica/filter1.npy')
m=np.load('/home/attale00/Desktop/classifiers/ica/meanI1.npy')
X1=X-m
data=np.dot(X1,W.T)
for i in range(len(testSet.data)):
testSet.data[i].extend(data[i,:])
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
clfPath = '/home/attale00/Desktop/classifiers/ica/rf128ICAHOGCOLOR'
f=file(clfPath,'r')
print 'classifier used: '+ f.name
clf = pickle.load(f)
testSet.classifiedAs=clf.predict(testSet.data)
testSet.probabilities=clf.predict_proba(testSet.data)
return testSet
def main(mode):
path = '/local/attale00/GoodPose'
path_ea = path+'/pascal128/'
fileNames = utils.getAllFiles(path+'/targets');
labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
#roi=(88,165,150,362)
roi=(44,84,88,168)
eM=np.load('/home/attale00/Desktop/mouthMask.npy')
m=cv2.resize(np.uint8(eM),(256,256));
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
dil = cv2.dilate(m,strel)
m=dil>0;
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(8,3))
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
_cross_validate(testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet)
else:
print 'not doing anything'
def main(mode):
path = '/local/attale00/extracted_pascal__4__Multi-PIE'
path_ea = path+'/color128/'
allLabelFiles = utils.getAllFiles('/local/attale00/a_labels')
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
#labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(50,74,96,160)
# X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)
#
#
# # perform ICA
# if mode not in ['s','v']:
# ica = FastICA(n_components=50,whiten=True)
# ica.fit(X)
# meanI=np.mean(X,axis=0)
# X1=X-meanI
# data=ica.transform(X1)
# filters=ica.components_
#
# elif mode in ['s','v']:
# W=np.load('/home/attale00/Desktop/classifiers/thesis/filter3.npy')
# m=np.load('/home/attale00/Desktop/classifiers/thesis/meanI3.npy')
# X1=X-m
# data=np.dot(X1,W.T)
#
# for i in range(len(testSet.data)):
# testSet.data[i].extend(data[i,:])
###
#
#
#
#
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(6,2),maskFromAlpha=False)
fg.getPoseLabel(testSet,pathToPoseFiles='/local/attale00/poseLabels/multipie/')
#fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=40)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
obj=classifierUtils.classifyWithOld(path,testSet,mode,clfPath = '/home/attale00/Desktop/classifiers/thesis/texture/hog_pose')
pickle.dump(obj,open('hog_pose','w'))
elif mode in ['c']:
print 'cross validation of data'
rValues = classifierUtils.dissectedCV(rf,testSet)
pickle.dump(rValues,open('texture_mp_','w'))
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet)
else:
print 'not doing anything'
def main(nJobs = 1):
path = '/local/attale00/GoodPose'
path_ea = path+'/extracted_alpha'
path_adata = path_ea + '/a_data'
filenames = utils.getAllFiles(path+'/targets');
attribute = 'mouth'
attribute_values = utils.parseLabelINIFile(path+'/mouth_labels/labels.ini',attribute);
print('------------Attribute: \t'+attribute+' ---------------')
for i in attribute_values:
print('Value: \t'+i)
print('----------------------------')
print('----------parsing label files------')
labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',filenames,cutoffSeq='.png',suffix='_face0.labels')
print('-----computing Features-----')
#make 10 bin hist for each mouth
mouthSet = fg.getHistogram(10,(0,270,60,452),hrange=(155.,255.0),labelFileDict = labs,path = path_ea+'/grayScale/',ending='_0.png')
mouthSet.targetNum = map(utils.mapMouthLabels2Two,mouthSet.target)
fpr=[]
tpr=[]
fnr = []
for i in xrange(1,101,5):
trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=.6)
(a,b,c)=_classify(trainingSet, testSet,n_estimators=i)
tpr.append(a)
fpr.append(b)
fnr.append(c)
plt.figure()
plt.plot(range(1,101,5),tpr)
plt.xlabel('number of trees')
plt.ylabel('score')
plt.title('.6 training set, max_feature 3, min_split 1' )
fpr=[]
tpr=[]
fnr = []
for i in xrange(1,20):
trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=i*.05)
(a,b,c)=_classify(trainingSet, testSet,n_estimators=70)
tpr.append(a)
fpr.append(b)
fnr.append(c)
plt.figure()
plt.plot(np.arange(0.05,1,0.05),tpr)
plt.xlabel('fraction used as training set')
plt.ylabel('score')
plt.title('ntrees = 70, max_feature 3, min_split 1' )
fpr=[]
tpr=[]
fnr = []
for i in xrange(1,11):
trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=.7)
(a,b,c)=_classify(trainingSet, testSet,n_estimators=70,max_features=i)
tpr.append(a)
fpr.append(b)
fnr.append(c)
plt.figure()
plt.plot(range(1,11),tpr)
plt.xlabel('number of features used')
plt.ylabel('score')
plt.title('ntrees = 70,frac=.7 min_split 1' )
plt.show()
#classifier
#linSVM = svm.SVC(kernel = 'linear',C=1)
#this takes forever: check if that can be true
#scoresLinSVM = cross_validation.cross_val_score(linSVM,data,y=targetNum,n_jobs=-1,verbose = 1)
#implement random forest classifier with verbosity level
#scoresRF = cross_validation.cross_val_score(rf,mouthSet.data,y=mouthSet.targetNum,n_jobs=nJobs,verbose = 1)
#print(scoresRF)
return
def main(mode):
labelFiles='/local/attale00/aflw_original_labels'
path_ea = '/local/attale00/AFLW_cropped/eyes/'
fileNames = utils.getAllFiles(path_ea)
goods = []
for f in fileNames:
im = cv2.imread(path_ea+f)
if im.shape[0] == 40 and im.shape[1] ==120:
goods.append(f)
fileNames = goods
labs=utils.parseLabelFiles(labelFiles,'glasses',fileNames,cutoffSeq='.png',suffix='.labels')
testSet = fg.dataContainer(labs)
#
# X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
#
#
# # perform ICA
# if mode not in ['s','v']:
# ica = FastICA(n_components=100,whiten=True)
# ica.fit(X)
# meanI=np.mean(X,axis=0)
# X1=X-meanI
# data=ica.transform(X1)
# filters=ica.components_
#
# elif mode in ['s','v']:
# W=np.load('/home/attale00/Desktop/classifiers/patches/filter2.npy')
# m=np.load('/home/attale00/Desktop/classifiers/patches/meanI2.npy')
# X1=X-m
# data=np.dot(X1,W.T)
#
# for i in range(len(testSet.fileNames)):
# testSet.data[i].extend(data[i,:])
#
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
fg.getHogFeature(testSet,None,path=path_ea,ending='.png',extraMask = None,orientations = 5, pixels_per_cell=(24,8),cells_per_block=(3,3),maskFromAlpha=False)
#testSet.targetNum=map(lambda x: 1 if x=='light' else 0,testSet.target)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
obj=classifierUtils.classifyWithOld(path,testSet,mode,clfPath = '/home/attale00/Desktop/classifiers/thesis/color_only')
#pickle.dump(obj,open('color_only','w'))
elif mode in ['c']:
print 'cross validation of data'
testSet.targetNum=map(lambda x: 1 if x=='1' else 0,testSet.target)
rf=classifierUtils.standardRF(max_features = 27,min_split=5,max_depth=60,n_estimators = 500)
rValues = classifierUtils.dissectedCV(rf,testSet)
plt.title('No glasses against rest')
plt.show()
#pickle.dump(rValues,open('patches_mp_','w'))
elif mode in ['save']:
print 'saving new classifier'
testSet.targetNum=map(lambda x: 1 if x=='1' else 0,testSet.target)
rf=classifierUtils.standardRF(max_features = 27,min_split=5,max_depth=60,n_estimators = 500)
_saveRF(testSet,rf)
else:
print 'not doing anything'
def main(mode):
path = '/local/attale00/AFLW_ALL'
path_ea = path+'/color128/'
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
testSetMirror = fg.dataContainer(labs)
for f in range(len(testSetMirror.fileNames)):
testSetMirror.fileNames[f]+='M'
roi=(50,74,96,160)
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)
Y=fg.getAllImagesFlat(path+'/mirror128/',testSet.fileNames,(128,256),roi=roi)
Z=np.concatenate((X,Y),axis=0)
# perform ICA
ica = FastICA(n_components=100,whiten=True)
ica.fit(Z)
meanI=np.mean(Z,axis=0)
X1=X-meanI
Y1=Y-meanI
data=ica.transform(X1)
datam=ica.transform(Y1)
filters=ica.components_
for i in range(len(fileNames)):
testSet.data[i].extend(data[i,:])
testSetMirror.data[i].extend(datam[i,:])
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
#fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)
#pca
# n_samples, n_features = X.shape
#
# mean_ = np.mean(X, axis=0)
# X -= mean_
# U, S, V = linalg.svd(X)
# explained_variance_ = (S ** 2) / n_samples
# explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
# K=V / S[:, np.newaxis] * np.sqrt(n_samples)
# filters=K[:100]
# data=np.dot(X,filters.T)
testSet.addContainer(testSetMirror)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
classifierUtils.dissectedCV(rf,testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf,filters=filters,meanI=meanI)
else:
print 'not doing anything'
def main(mode):
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#
fileNames = utils.getAllFiles(path_ea);
# minr = 10000;
# for f in fileNames:
# im = cv2.imread(path_ea+f,-1)
# if im.shape[0]!=40 or im.shape[1]!=120:
# print f
# print im.shape
# minr = minr if im.shape[0]>= minr else im.shape[0]
#
# print minr
#
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
roi=(0,37,0,115)
roi=None
#roi=(44,84,88,168)
# eM=np.load('/home/attale00/Desktop/mouthMask.npy')
# m=cv2.resize(np.uint8(eM),(256,256));
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# dil = cv2.dilate(m,strel)
#
# m=dil>0;
#X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
# perform ICA
names_open = []
names_closed = []
for i,f in enumerate(testSet.fileNames):
if testSet.targetNum[i] == 0:
names_closed.append(f)
elif testSet.targetNum[i] == 1:
names_open.append(f)
Xopen = fg.getAllImagesFlat(path_ea,names_open,(40,120))
XClosed = fg.getAllImagesFlat(path_ea,names_closed,(40,120))
if mode not in ['s','v']:
icaopen = FastICA(n_components=100,whiten=True)
icaopen.fit(Xopen)
meanIopen=np.mean(Xopen,axis=0)
X1open=Xopen-meanIopen
dataopen=icaopen.transform(X1open)
filtersopen=icaopen.components_
plottingUtils.showICAComponents(filtersopen,(40,120),4,4)
icaclosed = FastICA(n_components=100,whiten=True)
icaclosed.fit(XClosed)
meanIclosed=np.mean(XClosed,axis=0)
X1closed=XClosed-meanIclosed
dataclosed=icaclosed.transform(X1closed)
filtersclosed=icaclosed.components_
plottingUtils.showICAComponents(filtersclosed,(40,120),4,4)
plt.show()
elif mode in ['s','v']:
W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
X1=X-m
data=np.dot(X1,W.T)
for i in range(len(fileNames)):
testSet.data[i].extend(data[i,:])
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)
#fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
#pca
# n_samples, n_features = X.shape
#
# mean_ = np.mean(X, axis=0)
# X -= mean_
# U, S, V = linalg.svd(X)
# explained_variance_ = (S ** 2) / n_samples
# explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
# K=V / S[:, np.newaxis] * np.sqrt(n_samples)
# filters=K[:100]
# data=np.dot(X,filters.T)
rf=classifierUtils.standardRF(max_features = 27,min_split=13,max_depth=40)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
classifierUtils.dissectedCV(rf,testSet)
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf,filters=filters,meanI=meanI)
else:
print 'not doing anything'
from sklearn.pipeline import Pipeline
import featureGeneration as fg
import utils
from sklearn.decomposition import FastICA
import numpy as np
# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s')
###############################################################################
#data
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/cropped2/'
#
fileNames = utils.getAllFiles(path_ea);
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(0,37,0,115)
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(37,115),roi=roi)
#
def run(i,
data_path,
extension,
save_folder,
GRP=False,
compute_p=True,
calc_type='ksg',
use_population_parameters=False,
history_lengths=range(1, 6),
delays=range(1, 2),
**preprocessing_params):
"""
Run AIS calculation for a particular subject, using parameters which are either selected by
maximising the bias corrected average AIS for the individual subject, or parameters which are
first determined by averaging the AIS across the population
Arguments:
i -- An Int which states which file or subject to load and process
data_path -- Location of the data files
extension -- File extension of the data (eg. .csv, .tsv, .mat)
save_folder -- Subfolder of the 'Results' directory in which to save the local AIS values, parameters and p_values
GRP -- Set to True if processing the GRP data, which is one array of dimension (region, timepoints, subject)
compute_p -- If True, computes the p value of the returned AIS
calc_type -- The type of estimator to use for the JIDT calculator - 'gaussian' or 'ksg'
history_lengths -- Range of possible history length values
delays -- Range of possible delay values
preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
fcutlow, fcuthigh, use_filtfilt
"""
start_time = time.time()
files = utils.getAllFiles(data_path, extension)
if GRP:
file = files[0]
filename = '{:02}'.format(i) # Save the results by the subjects number
subject_id = i
else:
file = files[i]
filename = utils.basename(file)
subject_id = None
# Makes folders to save the results
os.makedirs("Results/{}/AIS/params".format(save_folder), exist_ok=True)
os.makedirs("Results/{}/AIS/p_values".format(save_folder), exist_ok=True)
print("Processing file {}: {}".format(i, filename))
if os.path.exists('Results/{}/AIS/p_values/{}_p.csv'.format(
save_folder, filename)):
print("Result already present")
exit() # If the results have already been saved, avoid running again
calc = startCalc(calc_type)
# Load and preprocess data
df = utils.loadData(file, subject_id=subject_id)
data = utils.preprocess(df, **preprocessing_params)
if use_population_parameters:
# If using population parameters, either load the pre-calculated parameters, or calculate them and save to file
if os.path.exists(
'Results/{}/AIS/population_params.csv'.format(save_folder)):
parameters = pd.read_csv(
'Results/{}/AIS/population_params.csv'.format(save_folder))
else:
parameters = getPopulationParameters(data_path, extension, calc,
history_lengths, delays,
**preprocessing_params)
pd.DataFrame(parameters, columns=['k', 'tau']).to_csv(
'Results/{}/AIS/population_params.csv'.format(save_folder),
index=None)
else:
parameters = None
results, all_parameters, p_values = getLocalsForAllRegions(
data,
calc,
history_lengths,
delays,
parameters,
print_max_idx=False,
compute_p=compute_p)
# Add back the trimmed sections at the start and end of the timeseries by padding with zeros
padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
preprocessing_params.get('trim_end', 0)))
results = np.pad(results, padding, mode='constant', constant_values=0)
# Save results
pd.DataFrame(results).to_csv('Results/{}/AIS/{}_AIS.csv'.format(
save_folder, filename),
index=None,
header=None)
params_df = pd.DataFrame(all_parameters, columns=['k', 'tau', 'DCE'])
params_df.to_csv('Results/{}/AIS/params/{}_params.csv'.format(
save_folder, filename),
index=None)
pd.DataFrame(p_values).to_csv('Results/{}/AIS/p_values/{}_p.csv'.format(
save_folder, filename),
index=None,
header=None)
print("\nTime taken:", round((time.time() - start_time) / 60, 2), 'min')
def run(i,
data_path,
extension,
save_folder,
raw_save_root="/scratch/InfoDynFuncStruct/Mike/N-back/",
save_every=20,
GRP=False,
compute_p=True,
compress=False,
set_k_to_0=False,
calc_type='ksg',
use_source_embedding=False,
**preprocessing_params):
"""
Run TE calculation for a particular subject. Parameters are loaded from file, based on the AIS calculation, or set
to 0 if set_k_to_0 is True
Arguments:
i -- An Int which states which file or subject to load and process
data_path -- Location of the data files
extension -- File extension of the data (eg. .csv, .tsv, .mat)
save_folder -- Subfolder of the 'Results' directory in which to save the local AIS values, parameters and p_values
raw_save_root -- Location to save the raw local TE values (as a npz or npy file)
save_every -- None, or Int giving the number of regions to calculate before saving the current state of the results
GRP -- Set to True if processing the GRP data, which is one array of dimension (region, timepoints, subject)
compute_p -- If True, computes the p value of the returned AIS
compress -- If True, the raw TE values are saved as a compressed npz file instead of an npy file
set_k_to_0 -- If True, skip loading of k and l parameters, instead initialising the DataFrame to zeros
calc_type -- The type of estimator to use for the JIDT calculator - 'gaussian' or 'ksg'
use_source_embedding -- If True, load up the values for history length and delay of the source from the AIS
calculations. Otherwise the source_history_length and source_delay parameters are set to 1
preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
fcutlow, fcuthigh, use_filtfilt
"""
start_time = time.time()
files = utils.getAllFiles(data_path, extension)
if GRP:
file = files[0]
filename = '{:02}'.format(i) # Save the results by the subjects number
subject_id = i
else:
file = files[i]
filename = utils.basename(file)
subject_id = None
print("Processing file {}: {}".format(i, filename))
# Check for the presence of the current_idx file
# If it's not present, then either no calculations have been done, or the final results have already been saved
if os.path.isfile(
os.path.join(
raw_save_root, "Results/{}/TE/raw/{}_current_idx.txt".format(
save_folder, filename))):
# Load previous results, which are always saved in the uncompressed format
results = np.load(
os.path.join(
raw_save_root,
"Results/{}/TE/raw/{}.npy".format(save_folder, filename)))
p_values = np.load(
os.path.join(
raw_save_root, "Results/{}/TE/raw/p_values/{}_p.npy".format(
save_folder, filename)))
dce = np.load(
os.path.join(
raw_save_root, "Results/{}/TE/raw/dce/{}_dce.npy".format(
save_folder, filename)))
with open(
os.path.join(
raw_save_root,
"Results/{}/TE/raw/{}_current_idx.txt".format(
save_folder, filename)), 'r') as f:
idx_values = f.readline()
idx_values = list(map(int, idx_values.split(',')))
print("Loading previous results")
print("Starting from index", idx_values)
else:
results, p_values, dce, idx_values = None, None, None, None
# Check both compressed and uncompressed options. If this file exists but the current_idx file doesn't then the
# final results have already been saved. Exit to avoid running again
if glob.glob(
os.path.join(
raw_save_root,
"Results/{}/TE/raw/p_values/{}_p.np*".format(
save_folder, filename))):
print("Result already present")
exit()
# Load parameter file
param_file = 'Results/{}/AIS/params/{}_params.csv'.format(
save_folder, filename)
if set_k_to_0:
df = utils.loadData(file, get_params=False, subject_id=subject_id)
param_df = pd.DataFrame(np.zeros((len(df), 2), dtype=int),
columns=['k', 'tau'])
else:
df, param_df = utils.loadData(file,
get_params=True,
param_file=param_file,
subject_id=subject_id)
data = utils.preprocess(df, **preprocessing_params)
saver = TEResultSaver(filename, save_folder, raw_save_root)
calc = startCalc(calc_type)
# Do the calculations
results, p_values, dce = getLocalsForAllRegionPairs(
data,
param_df,
calc,
compute_p,
saver=saver,
save_every=save_every,
results=results,
p_values=p_values,
dce=dce,
idx_values=idx_values,
use_source_embedding=use_source_embedding)
# Save the final results
# Add back the trimmed sections at the start and end of the timeseries by padding with zeros
padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
preprocessing_params.get('trim_end', 0)))
saver.save_final_result(results,
p_values,
dce,
padding=padding,
compress=compress)
print("\nTime taken:", round((time.time() - start_time) / 60, 2), 'min')
# -*- coding: utf-8 -*-
"""
Created on Thu May 30 15:49:39 2013
@author: attale00
"""
import utils
import os
labelPath='/export/somalia/egger/aflw_all/quality/labels'
#sourcePath='/export/somalia/egger/aflw_all/montages'
sourcePath='/export/somalia/egger/aflw_all'
targetPath ='/local/attale00/AFLW_ALL/montage'
targetPath='/local/attale00/AFLW_ALL/original'
labelFiles=utils.getAllFiles(labelPath)
counter =0
for labelFile in labelFiles:
f=open(labelPath+'/'+labelFile)
lines = f.readlines()
if lines[1].startswith('fit=good') or lines[4].startswith('fit_mouth=good'):
counter +=1
os.symlink(sourcePath+'/'+labelFile[:-13]+'.png',targetPath+'/'+labelFile[:-13]+'.png')
print counter
import sys
import plottingUtils
import classifierUtils
from sklearn import svm
from sklearn import linear_model
from sklearn.decomposition import FastICA
from scipy import linalg
import matplotlib.pyplot as plt
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/multiPIE_cropped3/'
path_label = '/local/attale00/a_labels'
allLabelFiles = utils.getAllFiles(path_label)
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
def multiPiePose():
path_pose = '/local/attale00/poseLabels/multipie'
poseLabel = utils.parseLabelFiles(path_pose,'rotY',labeledImages,cutoffSeq='.png',suffix='.frog')
poses = [(k,np.double(v)*180/np.pi) for k,v in poseLabel.iteritems()]
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 28 15:38:38 2013
@author: attale00
"""
import cv2
import numpy as np
import utils
path = '/local/attale00/GoodPose'
pathg = path+'/extracted_alpha/grayScale'
files = utils.getAllFiles(pathg)
cv2.namedWindow('thresh')
big = np.zeros((512,2*512),dtype='uint8')
for f in files:
im = cv2.imread(pathg+'/'+f,-1)
big[:,0:512]=im
ret,imbw = cv2.threshold(im,155,255,cv2.THRESH_BINARY)
big[:,512:1024]=imbw
cv2.imshow('test',big)
cv2.waitKey(0)
print('done')
def main(nJobs = 1):
path = '/local/attale00'
fileNames = utils.getAllFiles(path+'/targets');
attribute = 'mouth'
attribute_values = utils.parseLabelINIFile(path+'/mouth_labels/labels.ini',attribute);
print('------------Attribute: \t'+attribute+' ---------------')
for i in attribute_values:
print('Value: \t'+i)
print('----------------------------')
print('----------parsing label files------')
labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',fileNames,cutoffSeq='.png',suffix='_face0.labels')
print('-----computing Features-----')
#make 10 bin hist for each mouth
#roi = (40,200,100,200)
roi = (50,190,110,402)
roi2=(128,256,0,256)
mouthSet = fg.dataContainer(labs)
#fg.getHistogram(20,roi,hrange=(0,255),dataC = mouthSet,path = path+'/extracted/gradients/Direction/',ending='_0.png')
eM=np.load('/home/attale00/Desktop/mouthMask.npy')
m=cv2.resize(np.uint8(eM),(256,256));
strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
dil = cv2.dilate(m,strel)
m=dil>0;
# em=m[roi[0]:roi[1],roi[2]:roi[3]]
# m= m !=True
fg.getHogFeature(mouthSet,roi2,path=path+'/targets/grayScaleSmall/',ending=None,extraMask = None)
#fg.getPixelValues(mouthSet,roi,path=path_ea+'/',ending='_0.png',mask =m, scaleFactor = 10)
#fg.getColorHistogram(mouthSet,roi,path=path_ea+'/',ending='_0.png',colorspace=None,range=(1.0,255.0),bins = 20)
mouthSet.targetNum=map(utils.mapMouthLabels2Two,mouthSet.target)
n_estimators = range(10,180,20);
max_features = range(2,22,2)
max_depth = range(5,40,5)
max_depth.append(100)
min_split = range(1,20,2)
score=[]
var = []
for n in n_estimators:
scoresRF = _crossValidate(mouthSet, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet.data[0])),min_split = 5)
score.append(scoresRF.mean())
var.append(scoresRF.std())
print scoresRF
plt.errorbar(n_estimators,score,yerr=var)
plt.xlabel('number of trees')
# plt.ylabel('cross val score')
# mouthSet2 = fg.dataContainer(labs)
# roi=(256,512,0,512)
# fg.getColorHistogram(mouthSet2,roi,path=path+'/targets/',ending=None,colorspace='lab',range=(1.,255.0),bins = 20)
# mouthSet2.targetNum=map(utils.mapMouthLabels2Two,mouthSet2.target)
# score=[]
# var = []
# for n in n_estimators:
# scoresRF = _crossValidate(mouthSet2, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet2.data[0])),min_split = 5)
#
# score.append(scoresRF.mean())
# var.append(scoresRF.std())
#
# print scoresRF
# plt.errorbar(n_estimators,score,yerr=var)
# plt.xlabel('number of trees')
# plt.ylabel('cross val score')
# fg.getColorHistogram(mouthSet,roi,path=path_ea+'/',ending='_0.png',colorspace='lab',range=(100.0,255.0),bins = 20)
#
# score=[]
# var = []
# for n in n_estimators:
# scoresRF = _crossValidate(mouthSet, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet.data[0])),min_split = 5)
#
# score.append(scoresRF.mean())
# var.append(scoresRF.std())
#
# print scoresRF
# plt.errorbar(n_estimators,score,yerr=var)
# plt.xlabel('number of trees')
# plt.ylabel('cross val score')
# plt.legend(['HOG','LAB','HOG+LAB'])
# plt.title('20bins')
plt.show()
#classifier
#linSVM = svm.SVC(kernel = 'linear',C=1)
#this takes forever: check if that can be true
#scoresLinSVM = cross_validation.cross_val_score(linSVM,data,y=targetNum,n_jobs=-1,verbose = 1)
#implement random forest classifier with verbosity level
# roi_narrow=(60,160,130,382)
# extraMask = np.load('/home/attale00/Desktop/emptyMouthMask.npy')
#
# fg.getMeanAndVariance(roi_narrow,mouthSet,path_ea+'/',extraMask = extraMask,ending='_0.png')
# scoresRF = _crossValidate(mouthSet,max_features = 13)
# print 'Orientation and mean and cov' +str(scoresRF)
return
def main(mode):
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#
fileNames = utils.getAllFiles(path_ea);
# minr = 10000;
# for f in fileNames:
# im = cv2.imread(path_ea+f,-1)
# if im.shape[0]!=40 or im.shape[1]!=120:
# print f
# print im.shape
# minr = minr if im.shape[0]>= minr else im.shape[0]
#
# print minr
#
labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
testSet = fg.dataContainer(labs)
roi=(0,37,0,115)
roi=None
filters = None
meanI = None
#roi=(44,84,88,168)
# eM=np.load('/home/attale00/Desktop/mouthMask.npy')
# m=cv2.resize(np.uint8(eM),(256,256));
# strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# dil = cv2.dilate(m,strel)
#
# m=dil>0;
components = 150
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
# X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(120,40),roi=roi,resizeFactor = .5)
#
# perform ICA
if mode not in ['s','v']:
ica = FastICA(n_components=components,whiten=True)
ica.fit(X)
meanI=np.mean(X,axis=0)
X1=X-meanI
data=ica.transform(X1)
filters=ica.components_
elif mode in ['s','v']:
W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
X1=X-m
data=np.dot(X1,W.T)
for i in range(len(fileNames)):
testSet.data[i].extend(data[i,:])
#orientations = 2
#strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)
#fg.getColorHistogram(testSet,(0,40,40,80),path=path_ea,ending='.png',colorspace='lab',bins=bins)
#fg.getImagePatchStat(testSet,path=path_ea,patchSize=(4,12))
#fg.getImagePatchStat(testSet,path='/local/attale00/AFLW_cropped/mouth_img_error/',patchSize=(4,12))
#pca
# n_samples, n_features = X.shape
#
# mean_ = np.mean(X, axis=0)
# X -= mean_
# U, S, V = linalg.svd(X)
# explained_variance_ = (S ** 2) / n_samples
# explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
# K=V / S[:, np.newaxis] * np.sqrt(n_samples)
# filters=K[:100]
# data=np.dot(X,filters.T)
testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
rf=classifierUtils.standardRF(max_features = 23,min_split=15,max_depth=70,n_estimators=150)
#rf=classifierUtils.standardRF(max_features = 5,min_split=12,max_depth=45)
#rf = svm.NuSVC()
#rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)
if mode in ['s','v']:
print 'Classifying with loaded classifier'
_classifyWithOld(path,testSet,mode)
elif mode in ['c']:
print 'cross validation of data'
rValues = classifierUtils.dissectedCV(rf,testSet)
pickle.dump(rValues,open('patches_cv_ica_{}'.format(components),'w'))
elif mode in ['save']:
print 'saving new classifier'
_saveRF(testSet,rf,filters=filters,meanI=meanI)
else:
print 'not doing anything'
def getPopulationParameters(data_path, extension, calc, history_lengths,
delays, **preprocessing_params):
"""
Finds the parameters for history length and delay by considering the AIS values which have been averaged over the population
AIS is calculated for each subject, for each combination of parameters and then averaged across the subjects
The history length and delay which gives the maximum averaged AIS is chosen as the population parameters
Arguments:
data_path -- Location of the data files
extension -- File extension of the data
calc -- The JIDT calculator
history_lengths -- Range of possible history length values
delays -- Range of possible delay values
preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
fcutlow, fcuthigh, use_filtfilt
Returns:
parameters -- A numpy array with two columns, containing the (history_length, delay) of each region
"""
print("Finding population parameters")
files = utils.getAllFiles(data_path, extension)
data_all = utils.loadData(files[0], return_all_subjects=True)
if data_all.ndim == 3:
# Check to see if it's the GRP data, which will be returned as a 3D array of (regions, time points, subjects)
number_of_subjects = data_all.shape[2]
else:
number_of_subjects = len(files)
regions = data_all.shape[0]
# Initialise
ais_values = np.empty(
(regions, number_of_subjects, len(history_lengths), len(delays)))
for i in range(number_of_subjects):
if data_all.ndim == 3:
data = data_all[:, :,
i] # If it's the GRP data, get the data for each subject by slicing on the 3rd dimension
elif i == 0:
data = data_all
else: # If it's not the GRP data, just load the next file when the first one is done
data = utils.loadData(files[i])
data = utils.preprocess(data, **preprocessing_params)
dces = None # Just want to calculate the acls once for all the regions, so assign it as None first, and then replace it
for region in range(regions):
if dces is None:
dces = utils.getDCE(data)
ais_values[region, i] = runAllParameters(data[region], calc,
history_lengths, delays,
dces[region])
# Print progress bar
utils.update_progress(
(i * regions + region) / (number_of_subjects * regions),
end_text=" Subject {} of {}".format(i + 1,
number_of_subjects))
parameters = np.zeros((regions, 2), dtype=int)
mean_ais_values = ais_values.mean(axis=1) # Average over the subjects
for region in range(regions):
max_idx = utils.getMaxIdx2D(
mean_ais_values[region],
print_=False) # Find the argmax across the averaged AIS values
history_length = history_lengths[max_idx[0]]
delay = delays[max_idx[1]]
parameters[region] = (history_length, delay)
return parameters