def main():
    path='/local/attale00/'
    
    allFiles = utils.getAllFiles(path+'Multi-PIE/extracted')
    
    allLabelFiles = utils.getAllFiles(path+'Multi-PIE/db_labels')
    np.random.shuffle(allLabelFiles)
    #get the labels from the database, for each person
    sliceat=250
    labelstest=utils.parseLabelFiles(path+'Multi-PIE/db_labels','sex',allLabelFiles[0:sliceat],cutoffSeq='',suffix='')
    
    labelstraining = utils.parseLabelFiles(path+'Multi-PIE/db_labels','sex',allLabelFiles[sliceat:],cutoffSeq='',suffix='')
    #now generate the label dict for each file
    
    labsTest={}
    labsTraining={}
    for f in allFiles:
        if labelstest.has_key(f[0:3]+'.labels'):
            labsTest[f]=labelstest[f[0:3]+'.labels']
        elif labelstraining.has_key(f[0:3]+'.labels'):
            labsTraining[f]=labelstraining[f[0:3]+'.labels' ]
            
    testSet = fg.dataContainer(labsTest)
    trainingSet = fg.dataContainer(labsTraining)
    
    roi=(0,64,0,64)
    ppc=(8,8)
    cpb=(8,8)
    
    fg.getHogFeature(testSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None,pixels_per_cell=ppc,cells_per_block=cpb)
    
    fg.getHogFeature(trainingSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None,pixels_per_cell=ppc, cells_per_block=cpb)
    
    testSet.targetNum=map(utils.mapSexLabel2Two,testSet.target)
    trainingSet.targetNum = map(utils.mapSexLabel2Two,trainingSet.target)
    
    rf1=classifierUtils.standardRF(max_features=np.sqrt(len(testSet.data[0])))
    rf2=classifierUtils.standardRF(max_features=np.sqrt(len(trainingSet.data[0])))
    
    rf1.fit(testSet.data,testSet.targetNum)
    
    s=rf1.score(trainingSet.data,trainingSet.targetNum)
    trainingSet.classifiedAs=rf1.predict(trainingSet.data)
    trainingSet.hasBeenClassified=True
    classifierUtils.evaluateClassification(trainingSet,{0:'male',1:'female'})
    
    
    
    print 'Score: {}'.format(s)
    
    print '----------other way around ----\n'
    
    rf2.fit(trainingSet.data,trainingSet.targetNum)
    
    s=rf2.score(testSet.data,testSet.targetNum)
    testSet.classifiedAs=rf2.predict(testSet.data)
    testSet.hasBeenClassified=True
    classifierUtils.evaluateClassification(testSet,{0:'male',1:'female'})
    print 'Score: {}'.format(s)
def plotExcerpt(path='/local/attale00/GoodPose/extracted_alpha/grayScale',roi=(0,250,60,452),showDiff=''):


    files = utils.getAllFiles(path)
    im = cv2.imread(path+'/'+files[0],-1)    
    sh=np.shape(im)
    cv2.namedWindow('Panorama')
    big = np.zeros((sh[0],sh[1]*2),dtype='uint8')
    for f in files:
        im = cv2.imread(path+'/'+f,-1)
        try:
            im = cv2.cvtColor(im,cv2.cv.CV_RGB2GRAY)
        except Exception as e:
            pass
                
        big[:,0:sh[1]]=im
        cv2.rectangle(big,(roi[2],roi[1]),(roi[3],roi[0]),255,2)
        excerpt = im[roi[0]:roi[1],roi[2]:roi[3]]
    
        if showDiff in ['y','Y']:
            excerpt = np.diff(excerpt,n=1,axis=0)
        if showDiff in ['X','x']:
            excerpt = np.diff(excerpt,n=1,axis=1)
        big[roi[0]:roi[1],sh[1]+roi[2]:sh[1]+roi[3]]=excerpt
        cv2.imshow('Panorama',big)
        k=cv2.waitKey(0)
        if k==27:
            break
    cv2.destroyAllWindows()

    print('done')
def main(mode):
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#    
    fileNames = utils.getAllFiles(path_ea);

    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    fg_mode = 0
    size=(4,12)
    overlap=2
    #size=(40,120)
    fg.getImagePatchStat(testSet,path=path_ea,patchSize=size,overlap = overlap,mode=fg_mode)
    print 'feature vector length: {}'.format(len(testSet.data[0]))

    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=13,max_depth=40)
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        rValues = classifierUtils.dissectedCV(rf,testSet)
        pickle.dump(rValues,open('errorpatch_size_{}'.format(size[0]),'w'))
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf)
    else:
        print 'not doing anything'
def errorPatch():
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/mouth_img_error_multiPie/'
    
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
        
    
#    
    fileNames = labeledImages;

   
    
    
    testSet = fg.dataContainer(labs)
    
    
 
    fg.getImagePatchStat(testSet,path=path_ea,patchSize=(4,12),overlap = 2)
  
 
    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)

    clfPath = '/home/attale00/Desktop/classifiers/errorpatches/rferror'
    f=file(clfPath,'r')
    print 'classifier used: '+ f.name
    clf = pickle.load(f)
    testSet.classifiedAs=clf.predict(testSet.data)
    testSet.probabilities=clf.predict_proba(testSet.data)      
    return testSet
def main(mode):
    path = "/local/attale00/extracted_pascal__4__Multi-PIE"
    path_ea = path + "/color128/"

    allLabelFiles = utils.getAllFiles("/local/attale00/a_labels")

    labeledImages = [i[0:16] + ".png" for i in allLabelFiles]

    # labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    labs = utils.parseLabelFiles(
        "/local/attale00/a_labels", "mouth", labeledImages, cutoffSeq=".png", suffix="_face0.labels"
    )

    testSet = fg.dataContainer(labs)
    roi = (50, 74, 96, 160)
    X = fg.getAllImagesFlat(path_ea, testSet.fileNames, (128, 256), roi=roi)

    # perform ICA
    if mode not in ["s", "v"]:
        ica = FastICA(n_components=100, whiten=True)
        ica.fit(X)
        meanI = np.mean(X, axis=0)
        X1 = X - meanI
        data = ica.transform(X1)
        filters = ica.components_

    elif mode in ["s", "v"]:
        W = np.load("/home/attale00/Desktop/classifiers/ica/filter1.npy")
        m = np.load("/home/attale00/Desktop/classifiers/ica/meanI1.npy")
        X1 = X - m
        data = np.dot(X1, W.T)

    for i in range(len(testSet.data)):
        testSet.data[i].extend(data[i, :])

    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))

    # fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
    # fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)
    testSet.targetNum = map(utils.mapMouthLabels2Two, testSet.target)

    rf = classifierUtils.standardRF(max_features=np.sqrt(len(testSet.data[0])), min_split=5, max_depth=40)
    if mode in ["s", "v"]:
        print "Classifying with loaded classifier"
        classifierUtils.classifyWithOld(
            path, testSet, mode, clfPath="/home/attale00/Desktop/classifiers/ica/rf128ICA_1"
        )
    elif mode in ["c"]:
        print "cross validation of data"
        print "Scores"
        # print classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
        # _cvDissect(testSet,rf)
        classifierUtils.dissectedCV(rf, testSet)
        print "----"

    elif mode in ["save"]:
        print "saving new classifier"
        _saveRF(testSet)
    else:
        print "not doing anything"
def main(mode):
    path = '/local/attale00/extracted_pascal__4__Multi-PIE'
    path_ea = path+'/color128/'
   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    #labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    
    
    #fileNames = utils.getAllFiles(path_ea);
    
    
    
    
    #labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    
    
    roi=(50,74,96,160)
    #roi=(44,84,88,168)    
    
    
#    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
#    m=cv2.resize(np.uint8(eM),(256,256));
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#    dil = cv2.dilate(m,strel)
#    
#    m=dil>0;

    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))

  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 8, cells_per_block=(6,2),maskFromAlpha=False)
    fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,'s')
    elif mode in ['c']:
        print 'cross validation of data'
        print 'Scores'
        #print classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
        #_cvDissect(testSet,rf)
        classifierUtils.dissectedCV(rf,testSet)        
        print '----'
       
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet)
    else:
        print 'not doing anything'
def main(mode):
    path='/local/attale00/'
    
    allFiles = utils.getAllFiles(path+'Multi-PIE/extracted')
    
    allLabelFiles = utils.getAllFiles(path+'Multi-PIE/labels')
    #allLabelFiles =  utils.getAllFiles(path+'a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    #labs=utils.parseLabelFiles(path+'a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    
        
    
    testSet = fg.dataContainer(labs)
    
    
    roi = (0,32,0,64)
    #roi = (128,256,0,256)    
    
    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
    m=cv2.resize(np.uint8(eM),(256,256));
    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    dil = cv2.dilate(m,strel)
    
    m=dil>0;

  
    fg.getHogFeature(testSet,roi,path=path+'Multi-PIE_grayScale64/',ending=None,extraMask = None)
    fg.getColorHistogram(testSet,(50,190,110,402),path = path+'/Multi-PIE/extracted/',ending=None,colorspace='lab',range=(1.,255.0),bins = 20)
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    #testSet.targetNum = map(utils.mapGlassesLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        classifierUtils.dissectedCV(rf,testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf)
    else:
        print 'not doing anything'
Exemple #8
0
    def run_individual_parameters(i,
                                  data_path,
                                  extension,
                                  save_folder,
                                  GRP=False,
                                  compute_p=True,
                                  **preprocessing_params):
        """
        Arguments:
            GRP -- True if processing the GRP data
        """
        files = utils.getAllFiles(data_path, extension)
        if GRP:
            file = files[0]
            filename = '{:02}'.format(
                i)  # Save the results by the subjects number
            subject_id = i
        else:
            file = files[i]
            filename = utils.basename(file)
            subject_id = None
        os.makedirs("Results/{}/AM/idx".format(save_folder), exist_ok=True)
        os.makedirs("Results/{}/AM/p_values".format(save_folder),
                    exist_ok=True)

        print("Processing", i, ":", filename)
        if os.path.exists('Results/{}/AM/p_values/{}.csv'.format(
                save_folder, filename)):
            exit()

        param_file = 'Results/{}/AIS/idx/{}.csv'.format(save_folder, filename)
        df, param_df = utils.loadData(file,
                                      get_params=True,
                                      param_file=param_file,
                                      subject_id=subject_id)
        data = utils.preprocess(df, **preprocessing_params)
        results, p_values = getLocalsForAllRegions(data,
                                                   param_df,
                                                   compute_p=compute_p)
        # Add back the trimmed sections
        padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
                            preprocessing_params.get('trim_end', 0)))
        results = np.pad(results, padding, mode='constant', constant_values=0)

        pd.DataFrame(results).to_csv('Results/{}/AM/{}_AM.csv'.format(
            save_folder, filename),
                                     index=None,
                                     header=None)
        pd.DataFrame(p_values).to_csv('Results/{}/AM/p_values/{}.csv'.format(
            save_folder, filename),
                                      index=None,
                                      header=None)
        try:
            utils.plotHeatmap(results, divergent=True)
        except:
            pass
def patches():
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/multiPIE_cropped3/'
    

   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
   
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')

    fileNames = labeledImages;
    testSet = fg.dataContainer(labs)
    
    
    roi=(0,37,0,115)
    roi=None    
 


            
 
    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
 
   
    
    W=np.load('/home/attale00/Desktop/classifiers/patches/filter2.npy')
    m=np.load('/home/attale00/Desktop/classifiers/patches/meanI2.npy')
    X1=X-m
    data=np.dot(X1,W.T)    
    
    for i in range(len(fileNames)):
            testSet.data[i].extend(data[i,:])

    
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, pixels_per_cell=(24,8),cells_per_block=(3,3),maskFromAlpha=False)
    fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)
   
   
            
    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    clfPath = '/home/attale00/Desktop/classifiers/patches/rfICAHogColor'
    f=file(clfPath,'r')
    print 'classifier used: '+ f.name
    clf = pickle.load(f)
    testSet.classifiedAs=clf.predict(testSet.data)
    testSet.probabilities=clf.predict_proba(testSet.data)      
        

    
    return testSet
def main(mode):
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#    
    fileNames = utils.getAllFiles(path_ea);

    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    testSet = fg.dataContainer(labs)
    components = 150
    roi=None
    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
#    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(120,40),roi=roi,resizeFactor = .5)
# 
# perform ICA
    if mode not in ['s','v']:
        ica = FastICA(n_components=components,whiten=True)
        ica.fit(X)
        meanI=np.mean(X,axis=0)
        X1=X-meanI
        data=ica.transform(X1)
        filters=ica.components_
        
    elif mode in ['s','v']:
        W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
        m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
        X1=X-m
        data=np.dot(X1,W.T)    
    
    for i in range(len(fileNames)):
            testSet.data[i].extend(data[i,:])
            
    print 'feature vector length: {}'.format(len(testSet.data[0]))

    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=13,max_depth=40)
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        rValues = classifierUtils.dissectedCV(rf,testSet)
        pickle.dump(rValues,open('errorpatch_ica','w'))
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf,filters=filters,meanI=meanI)
    else:
        print 'not doing anything'
Exemple #11
0
def main(nJobs = 1):

    path = '/local/attale00/GoodPose/extracted_alpha/grayScale64'
    
    fileNames = utils.getAllFiles(path);
    

    labs=utils.parseLabelFiles('/local/attale00/GoodPose'+'/mouth_labels','mouth',fileNames,cutoffSeq='_0.png',suffix='_face0.labels')
    print('-----computing Features-----')

    roi2 = (0,32,0,64)
    mouthSet = fg.dataContainer(labs)

    #load the mask for the mouth room pixels and dilate it
    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
    m=cv2.resize(np.uint8(eM),(256,256));
    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    dil = cv2.dilate(m,strel)

    m=dil>0;

    #get the features
    fg.getHogFeature(mouthSet,roi2,path=path+'/',ending=None,extraMask = None)
    
    #map the string labels to numbers (required by sklearn)
    #change the mapping here for different classifiers
    mouthSet.targetNum=map(utils.mapMouthLabels2Two,mouthSet.target)
    n_estimators = 100
    min_split = 10
    max_depth = 20
    max_features = np.sqrt(len(mouthSet.data[0]))
    
    rf = classifierUtils.standardRF(max_features = max_features)
    rf2=classifierUtils.standardRF(max_features=max_features)
    
    score=classifierUtils.standardCrossvalidation(rf2,mouthSet)

    rf.fit(mouthSet.data,mouthSet.targetNum)
    
    pickle.dump(rf,open('/home/attale00/Desktop/classifiers/RandomForestMouthclassifier_12','w'))
    
    f=open('/home/attale00/Desktop/classifiers/RandomForestMouthclassifier_12.txt','w')
    f.write('Trained on aflw\n')
    f.write('Attribute: mouth' )
    f.write('Features: getHogFeature(mouthSet,roi2,path=path,ending=None,extraMask = m) on 64*64 grayScale 3 direction bins \n')
    f.write('ROI:(0,32,0,64)\n')
    f.write('labels: closed, narrow: 0, open, wideOpen: 1\n')
    f.write('CV Score: {}\n'.format(score))
    f.close()
Exemple #12
0
def aflwPose():

    path_ea = '/local/attale00/AFLW_cropped/mouth_img_error/'
#    
    fileNames = utils.getAllFiles(path_ea);

    
    
    path_pose = '/local/attale00/poseLabels/aflw'
    
    poseLabelaflw = utils.parseLabelFiles(path_pose,'rotY',fileNames,cutoffSeq='.png',suffix='.frog')
    
    poses_aflw = [np.double(i)*180/np.pi for i in poseLabelaflw.values()]
    
    

    return np.array(poses_aflw)
def main(mode):
    path = '/local/attale00/AFLW_ALL'
    path_ea = path+'/color256/'
    
    fileNames = utils.getAllFiles(path_ea);
    
  
    
    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')

    
    
    testSet = fg.dataContainer(labs)
    
    
    roi=(88,165,150,362)
    #roi=(44,84,88,168)    
    
    
#    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
#    m=cv2.resize(np.uint8(eM),(256,256));
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#    dil = cv2.dilate(m,strel)
#    
#    m=dil>0;

  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(26,9),maskFromAlpha=False)
    fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
    print len(testSet.data)
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        #classifierUtils.standardCrossvalidation(rf,testSet,n_jobs=5)
        classifierUtils.dissectedCV(rf,testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet)
    else:
        print 'not doing anything'
Exemple #14
0
def multiPiePose():

    

   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
        
   
    path_pose = '/local/attale00/poseLabels/multipie'
    
    poseLabel = utils.parseLabelFiles(path_pose,'rotY',labeledImages,cutoffSeq='.png',suffix='.frog')
    
    poses = np.array([np.double(i)*180/np.pi for i in poseLabel.values()])
    return poses
def plotThreshold(path='/local/attale00/GoodPose/extracted_alpha/grayScale',thresh=100):


    files = utils.getAllFiles(path)
   
    big = np.zeros((512,2*512),dtype='uint8')
    for f in files:
        im = cv2.imread(path+'/'+f,-1)
        big[:,0:512]=im
        ret,imbw = cv2.threshold(im,thresh,255,cv2.THRESH_BINARY)
        big[:,512:1024]=imbw
        cv2.imshow('test',big)
        k=cv2.waitKey(0)
        if k==27:
            
            return

    print('done')
def main(mode):
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#    
    fileNames = utils.getAllFiles(path_ea);

    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    roi=None
    testSet = fg.dataContainer(labs)
    testSetMirror = fg.dataContainer(labs)
    for f in range(len(testSetMirror.fileNames)):
        testSetMirror.fileNames[f]+='M'
    
  
    orientations = 9
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = orientations, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)

  
    fg.getHogFeature(testSetMirror,roi,path='/local/attale00/AFLW_cropped/mirrored/', ending='.png',orientations = orientations, cells_per_block=(3,3),pixels_per_cell=(24,8))    
    
    testSet.addContainer(testSetMirror)
  
 
    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = 30,min_split=12,max_depth=70)
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        rValues = classifierUtils.dissectedCV(rf,testSet)
        pickle.dump(rValues,open('patches_cv_hog_{}'.format(orientations),'w'))
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf)
    else:
        print 'not doing anything'
def plotExcerpt(path='/local/attale00/GoodPose/extracted_alpha/grayScale',roi=(0,250,60,452)):


    files = utils.getAllFiles(path)
    cv2.namedWindow('thresh')
    big = np.zeros((512,2*512),dtype='uint8')
    for f in files:
        im = cv2.imread(path+'/'+f,-1)
        im = cv2.cvtColor(im,cv2.cv.CV_RGB2GRAY)
        big[:,0:512]=im
        
        big[roi[0]:roi[1],512:512+roi[3]-roi[2]]=im[roi[0]:roi[1],roi[2]:roi[3]]
        cv2.imshow('test',big)
        k=cv2.waitKey(0)
        if k==27:
            return

    print('done')
    
def main(mode):
    path = '/local/attale00/extracted_pascal__4__Multi-PIE'
    path_ea = path+'/color256/'
   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    #labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    
    testSet = fg.dataContainer(labs)
    
    
    roi=(88,165,150,362)
    #roi=(44,84,88,168)    
    
    
#    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
#    m=cv2.resize(np.uint8(eM),(256,256));
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#    dil = cv2.dilate(m,strel)
#    
#    m=dil>0;

  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(26,9),maskFromAlpha=False)
    fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,'v')
    elif mode in ['c']:
        print 'cross validation of data'
        _cross_validate(testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet)
    else:
        print 'not doing anything'
def texture():
    path = '/local/attale00/extracted_pascal__4__Multi-PIE'
    path_ea = path+'/color128/'
   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    
        
    testSet = fg.dataContainer(labs)    
    roi=(50,74,96,160)
    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)


    W=np.load('/home/attale00/Desktop/classifiers/ica/filter1.npy')
    m=np.load('/home/attale00/Desktop/classifiers/ica/meanI1.npy')
    X1=X-m
    data=np.dot(X1,W.T)    
    
    for i in range(len(testSet.data)):
        testSet.data[i].extend(data[i,:])
    
 
  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
    fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    
       
    clfPath = '/home/attale00/Desktop/classifiers/ica/rf128ICAHOGCOLOR'
    f=file(clfPath,'r')
    print 'classifier used: '+ f.name
    clf = pickle.load(f)
    testSet.classifiedAs=clf.predict(testSet.data)
    testSet.probabilities=clf.predict_proba(testSet.data)      
        

    
    return testSet
def main(mode):
    path = '/local/attale00/GoodPose'
    path_ea = path+'/pascal128/'
    
    fileNames = utils.getAllFiles(path+'/targets');
    
    
    
    
    labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    
    
    #roi=(88,165,150,362)
    roi=(44,84,88,168)    
    
    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
    m=cv2.resize(np.uint8(eM),(256,256));
    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    dil = cv2.dilate(m,strel)
    
    m=dil>0;

  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(8,3))
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        _cross_validate(testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet)
    else:
        print 'not doing anything'
def main(mode):
    path = '/local/attale00/extracted_pascal__4__Multi-PIE'
    path_ea = path+'/color128/'
   
    allLabelFiles =  utils.getAllFiles('/local/attale00/a_labels')
    
    labeledImages = [i[0:16]+'.png' for i in allLabelFiles]
    
    #labs=utils.parseLabelFiles(path+'/Multi-PIE/labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    labs=utils.parseLabelFiles('/local/attale00/a_labels','mouth',labeledImages,cutoffSeq='.png',suffix='_face0.labels')
    
        
    testSet = fg.dataContainer(labs)    
    roi=(50,74,96,160)
#    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)
#
#    
#    # perform ICA
#    if mode not in ['s','v']:
#        ica = FastICA(n_components=50,whiten=True)
#        ica.fit(X)
#        meanI=np.mean(X,axis=0)
#        X1=X-meanI
#        data=ica.transform(X1)
#        filters=ica.components_
#        
#    elif mode in ['s','v']:
#        W=np.load('/home/attale00/Desktop/classifiers/thesis/filter3.npy')
#        m=np.load('/home/attale00/Desktop/classifiers/thesis/meanI3.npy')
#        X1=X-m
#        data=np.dot(X1,W.T)    
#    
#    for i in range(len(testSet.data)):
#        testSet.data[i].extend(data[i,:])
###    
#    
#   
#
#
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))

  
    fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 4, cells_per_block=(6,2),maskFromAlpha=False)
    fg.getPoseLabel(testSet,pathToPoseFiles='/local/attale00/poseLabels/multipie/')    
    #fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=40)    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        obj=classifierUtils.classifyWithOld(path,testSet,mode,clfPath = '/home/attale00/Desktop/classifiers/thesis/texture/hog_pose')
        pickle.dump(obj,open('hog_pose','w'))
    elif mode in ['c']:
        print 'cross validation of data'
        rValues = classifierUtils.dissectedCV(rf,testSet)
        pickle.dump(rValues,open('texture_mp_','w'))
       
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet)
    else:
        print 'not doing anything'
def main(nJobs = 1):

    path = '/local/attale00/GoodPose'
    path_ea = path+'/extracted_alpha'
    path_adata = path_ea + '/a_data'
    
    filenames = utils.getAllFiles(path+'/targets');
    
    attribute = 'mouth'
    
    attribute_values = utils.parseLabelINIFile(path+'/mouth_labels/labels.ini',attribute);
    
    print('------------Attribute: \t'+attribute+' ---------------')
    for i in attribute_values:
        print('Value: \t'+i)
        
    print('----------------------------')
    print('----------parsing label files------')
    labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',filenames,cutoffSeq='.png',suffix='_face0.labels')
    print('-----computing Features-----')
    #make 10 bin hist for each mouth
    mouthSet = fg.getHistogram(10,(0,270,60,452),hrange=(155.,255.0),labelFileDict = labs,path = path_ea+'/grayScale/',ending='_0.png')
    
    mouthSet.targetNum = map(utils.mapMouthLabels2Two,mouthSet.target)
    fpr=[]
    tpr=[]
    fnr = []    

    for i in xrange(1,101,5):
        trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=.6)
        (a,b,c)=_classify(trainingSet, testSet,n_estimators=i)
        tpr.append(a)
        fpr.append(b)
        fnr.append(c)
    
    plt.figure()
    plt.plot(range(1,101,5),tpr)
    plt.xlabel('number of trees')
    plt.ylabel('score')
    plt.title('.6 training set, max_feature 3, min_split 1' )
   
    fpr=[]
    tpr=[]
    fnr = []
    for i in xrange(1,20):
        trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=i*.05)
        (a,b,c)=_classify(trainingSet, testSet,n_estimators=70)
        tpr.append(a)
        fpr.append(b)
        fnr.append(c)
    
    plt.figure()
    plt.plot(np.arange(0.05,1,0.05),tpr)
    plt.xlabel('fraction used as training set')
    plt.ylabel('score')
    plt.title('ntrees = 70, max_feature 3, min_split 1' )
    
    fpr=[]
    tpr=[]
    fnr = []
    for i in xrange(1,11):
        trainingSet,testSet = mouthSet.splitInTestAndTraining(frac=.7)
        (a,b,c)=_classify(trainingSet, testSet,n_estimators=70,max_features=i)
        tpr.append(a)
        fpr.append(b)
        fnr.append(c)
    
    plt.figure()
    plt.plot(range(1,11),tpr)
    plt.xlabel('number of features used')
    plt.ylabel('score')
    plt.title('ntrees = 70,frac=.7 min_split 1' )
    
    
    plt.show()
    
    #classifier
    #linSVM = svm.SVC(kernel = 'linear',C=1)
    
    #this takes forever: check if that can be true
    #scoresLinSVM = cross_validation.cross_val_score(linSVM,data,y=targetNum,n_jobs=-1,verbose = 1)
    
    #implement random forest classifier with verbosity level
    
    #scoresRF = cross_validation.cross_val_score(rf,mouthSet.data,y=mouthSet.targetNum,n_jobs=nJobs,verbose = 1)
    #print(scoresRF)    
    return
def main(mode):
    labelFiles='/local/attale00/aflw_original_labels'
    path_ea = '/local/attale00/AFLW_cropped/eyes/'
    

    fileNames = utils.getAllFiles(path_ea)
    goods = []
    for f in fileNames:
        im = cv2.imread(path_ea+f)
        if im.shape[0] == 40 and im.shape[1] ==120:
            goods.append(f)
    
    fileNames = goods
    labs=utils.parseLabelFiles(labelFiles,'glasses',fileNames,cutoffSeq='.png',suffix='.labels')
    
    
   
    
    
    testSet = fg.dataContainer(labs)
    
    


            
# 
#    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
# 
#        
#    # perform ICA
#    if mode not in ['s','v']:
#        ica = FastICA(n_components=100,whiten=True)
#        ica.fit(X)
#        meanI=np.mean(X,axis=0)
#        X1=X-meanI
#        data=ica.transform(X1)
#        filters=ica.components_
#        
#    elif mode in ['s','v']:
#        W=np.load('/home/attale00/Desktop/classifiers/patches/filter2.npy')
#        m=np.load('/home/attale00/Desktop/classifiers/patches/meanI2.npy')
#        X1=X-m
#        data=np.dot(X1,W.T)    
#    
#    for i in range(len(testSet.fileNames)):
#            testSet.data[i].extend(data[i,:])
#
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    fg.getHogFeature(testSet,None,path=path_ea,ending='.png',extraMask = None,orientations = 5, pixels_per_cell=(24,8),cells_per_block=(3,3),maskFromAlpha=False)

    #testSet.targetNum=map(lambda x: 1 if x=='light' else 0,testSet.target)
   
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        obj=classifierUtils.classifyWithOld(path,testSet,mode,clfPath = '/home/attale00/Desktop/classifiers/thesis/color_only')       
        #pickle.dump(obj,open('color_only','w'))
    elif mode in ['c']:
        print 'cross validation of data'
        
        
        testSet.targetNum=map(lambda x: 1 if x=='1' else 0,testSet.target)
        rf=classifierUtils.standardRF(max_features = 27,min_split=5,max_depth=60,n_estimators = 500)
        rValues = classifierUtils.dissectedCV(rf,testSet)
        plt.title('No glasses against rest')
        plt.show()
        
        #pickle.dump(rValues,open('patches_mp_','w'))
    elif mode in ['save']:
        print 'saving new classifier'
        testSet.targetNum=map(lambda x: 1 if x=='1' else 0,testSet.target)
        rf=classifierUtils.standardRF(max_features = 27,min_split=5,max_depth=60,n_estimators = 500)
        _saveRF(testSet,rf)
    else:
        print 'not doing anything'
def main(mode):
    path = '/local/attale00/AFLW_ALL'
    path_ea = path+'/color128/'
    
    fileNames = utils.getAllFiles(path_ea);
    
    
    
    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    
    testSetMirror = fg.dataContainer(labs)
    for f in range(len(testSetMirror.fileNames)):
        testSetMirror.fileNames[f]+='M'
    
    
    roi=(50,74,96,160)
 
 

    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(128,256),roi=roi)
    Y=fg.getAllImagesFlat(path+'/mirror128/',testSet.fileNames,(128,256),roi=roi)
    Z=np.concatenate((X,Y),axis=0)
    # perform ICA
    ica = FastICA(n_components=100,whiten=True)
    ica.fit(Z)
    meanI=np.mean(Z,axis=0)
    X1=X-meanI
    Y1=Y-meanI    
    data=ica.transform(X1)
    datam=ica.transform(Y1)
    filters=ica.components_
    for i in range(len(fileNames)):
        testSet.data[i].extend(data[i,:])
        testSetMirror.data[i].extend(datam[i,:])


    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    #fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 3, cells_per_block=(6,2),maskFromAlpha=False)
    #fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=10)

  
    #pca
#    n_samples, n_features = X.shape
#
#    mean_ = np.mean(X, axis=0)
#    X -= mean_
#    U, S, V = linalg.svd(X)
#    explained_variance_ = (S ** 2) / n_samples
#    explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
#    K=V / S[:, np.newaxis] * np.sqrt(n_samples)
#    filters=K[:100]
#    data=np.dot(X,filters.T)    
    
    testSet.addContainer(testSetMirror)
            
    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = np.sqrt(len(testSet.data[0])),min_split=5,max_depth=40)
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        classifierUtils.dissectedCV(rf,testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf,filters=filters,meanI=meanI)
    else:
        print 'not doing anything'
def main(mode):
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#    
    fileNames = utils.getAllFiles(path_ea);
#    minr = 10000;
#    for f in fileNames:
#        im = cv2.imread(path_ea+f,-1)
#        if im.shape[0]!=40 or im.shape[1]!=120:
#            print f
#            print im.shape
#        minr = minr if im.shape[0]>= minr else im.shape[0]
#    
#    print minr
#    
    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    
    roi=(0,37,0,115)
    roi=None    
    #roi=(44,84,88,168)    
    
    
#    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
#    m=cv2.resize(np.uint8(eM),(256,256));
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#    dil = cv2.dilate(m,strel)
#    
#    m=dil>0;


            
 
    #X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
   
# perform ICA 
    names_open = []
    names_closed = []
    for i,f in enumerate(testSet.fileNames):
        if testSet.targetNum[i] == 0:
            names_closed.append(f)
        elif testSet.targetNum[i] == 1:
            names_open.append(f)
    Xopen = fg.getAllImagesFlat(path_ea,names_open,(40,120))
    XClosed = fg.getAllImagesFlat(path_ea,names_closed,(40,120))
    
    if mode not in ['s','v']:
        icaopen = FastICA(n_components=100,whiten=True)
        icaopen.fit(Xopen)
        meanIopen=np.mean(Xopen,axis=0)
        X1open=Xopen-meanIopen
        dataopen=icaopen.transform(X1open)
        filtersopen=icaopen.components_
        plottingUtils.showICAComponents(filtersopen,(40,120),4,4)
        icaclosed = FastICA(n_components=100,whiten=True)
        icaclosed.fit(XClosed)
        meanIclosed=np.mean(XClosed,axis=0)
        X1closed=XClosed-meanIclosed
        dataclosed=icaclosed.transform(X1closed)
        filtersclosed=icaclosed.components_        
        plottingUtils.showICAComponents(filtersclosed,(40,120),4,4)
        
        plt.show()
        
    elif mode in ['s','v']:
        W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
        m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
        X1=X-m
        data=np.dot(X1,W.T)    
    
    for i in range(len(fileNames)):
            testSet.data[i].extend(data[i,:])


    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    #fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)
    #fg.getColorHistogram(testSet,roi,path=path_ea,ending='.png',colorspace='lab',bins=20)

  
    #pca
#    n_samples, n_features = X.shape
#
#    mean_ = np.mean(X, axis=0)
#    X -= mean_
#    U, S, V = linalg.svd(X)
#    explained_variance_ = (S ** 2) / n_samples
#    explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
#    K=V / S[:, np.newaxis] * np.sqrt(n_samples)
#    filters=K[:100]
#    data=np.dot(X,filters.T)    
    
   
            
    
    
    rf=classifierUtils.standardRF(max_features = 27,min_split=13,max_depth=40)
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        classifierUtils.dissectedCV(rf,testSet)
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf,filters=filters,meanI=meanI)
    else:
        print 'not doing anything'
Exemple #26
0
from sklearn.pipeline import Pipeline
import featureGeneration as fg
import utils
from sklearn.decomposition import FastICA
import numpy as np 
# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')


###############################################################################
#data
path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/cropped2/'
#    
fileNames = utils.getAllFiles(path_ea);


labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')



testSet = fg.dataContainer(labs)


roi=(0,37,0,115)

 
X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(37,115),roi=roi)
 
#        
Exemple #27
0
def run(i,
        data_path,
        extension,
        save_folder,
        GRP=False,
        compute_p=True,
        calc_type='ksg',
        use_population_parameters=False,
        history_lengths=range(1, 6),
        delays=range(1, 2),
        **preprocessing_params):
    """
    Run AIS calculation for a particular subject, using parameters which are either selected by 
    maximising the bias corrected average AIS for the individual subject, or parameters which are
    first determined by averaging the AIS across the population
    
    Arguments:
        i -- An Int which states which file or subject to load and process
        data_path -- Location of the data files
        extension -- File extension of the data (eg. .csv, .tsv, .mat)
        save_folder -- Subfolder of the 'Results' directory in which to save the local AIS values, parameters and p_values
        GRP -- Set to True if processing the GRP data, which is one array of dimension (region, timepoints, subject)
        compute_p -- If True, computes the p value of the returned AIS
        calc_type -- The type of estimator to use for the JIDT calculator - 'gaussian' or 'ksg'
        history_lengths -- Range of possible history length values
        delays -- Range of possible delay values
        preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
                                Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
                                fcutlow, fcuthigh, use_filtfilt
    """
    start_time = time.time()
    files = utils.getAllFiles(data_path, extension)
    if GRP:
        file = files[0]
        filename = '{:02}'.format(i)  # Save the results by the subjects number
        subject_id = i
    else:
        file = files[i]
        filename = utils.basename(file)
        subject_id = None
    # Makes folders to save the results
    os.makedirs("Results/{}/AIS/params".format(save_folder), exist_ok=True)
    os.makedirs("Results/{}/AIS/p_values".format(save_folder), exist_ok=True)

    print("Processing file {}: {}".format(i, filename))
    if os.path.exists('Results/{}/AIS/p_values/{}_p.csv'.format(
            save_folder, filename)):
        print("Result already present")
        exit()  # If the results have already been saved, avoid running again

    calc = startCalc(calc_type)

    # Load and preprocess data
    df = utils.loadData(file, subject_id=subject_id)
    data = utils.preprocess(df, **preprocessing_params)

    if use_population_parameters:
        # If using population parameters, either load the pre-calculated parameters, or calculate them and save to file
        if os.path.exists(
                'Results/{}/AIS/population_params.csv'.format(save_folder)):
            parameters = pd.read_csv(
                'Results/{}/AIS/population_params.csv'.format(save_folder))
        else:
            parameters = getPopulationParameters(data_path, extension, calc,
                                                 history_lengths, delays,
                                                 **preprocessing_params)
            pd.DataFrame(parameters, columns=['k', 'tau']).to_csv(
                'Results/{}/AIS/population_params.csv'.format(save_folder),
                index=None)
    else:
        parameters = None

    results, all_parameters, p_values = getLocalsForAllRegions(
        data,
        calc,
        history_lengths,
        delays,
        parameters,
        print_max_idx=False,
        compute_p=compute_p)

    # Add back the trimmed sections at the start and end of the timeseries by padding with zeros
    padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
                        preprocessing_params.get('trim_end', 0)))
    results = np.pad(results, padding, mode='constant', constant_values=0)

    # Save results
    pd.DataFrame(results).to_csv('Results/{}/AIS/{}_AIS.csv'.format(
        save_folder, filename),
                                 index=None,
                                 header=None)
    params_df = pd.DataFrame(all_parameters, columns=['k', 'tau', 'DCE'])
    params_df.to_csv('Results/{}/AIS/params/{}_params.csv'.format(
        save_folder, filename),
                     index=None)
    pd.DataFrame(p_values).to_csv('Results/{}/AIS/p_values/{}_p.csv'.format(
        save_folder, filename),
                                  index=None,
                                  header=None)

    print("\nTime taken:", round((time.time() - start_time) / 60, 2), 'min')
Exemple #28
0
def run(i,
        data_path,
        extension,
        save_folder,
        raw_save_root="/scratch/InfoDynFuncStruct/Mike/N-back/",
        save_every=20,
        GRP=False,
        compute_p=True,
        compress=False,
        set_k_to_0=False,
        calc_type='ksg',
        use_source_embedding=False,
        **preprocessing_params):
    """
    Run TE calculation for a particular subject. Parameters are loaded from file, based on the AIS calculation, or set
    to 0 if set_k_to_0 is True
    
    Arguments:
        i -- An Int which states which file or subject to load and process
        data_path -- Location of the data files
        extension -- File extension of the data (eg. .csv, .tsv, .mat)
        save_folder -- Subfolder of the 'Results' directory in which to save the local AIS values, parameters and p_values
        raw_save_root -- Location to save the raw local TE values (as a npz or npy file)
        save_every -- None, or Int giving the number of regions to calculate before saving the current state of the results
        GRP -- Set to True if processing the GRP data, which is one array of dimension (region, timepoints, subject)
        compute_p -- If True, computes the p value of the returned AIS
        compress -- If True, the raw TE values are saved as a compressed npz file instead of an npy file
        set_k_to_0 -- If True, skip loading of k and l parameters, instead initialising the DataFrame to zeros
        calc_type -- The type of estimator to use for the JIDT calculator - 'gaussian' or 'ksg'
        use_source_embedding -- If True, load up the values for history length and delay of the source from the AIS
                                calculations. Otherwise the source_history_length and source_delay parameters are set to 1
        preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
                                Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
                                fcutlow, fcuthigh, use_filtfilt
    """
    start_time = time.time()
    files = utils.getAllFiles(data_path, extension)
    if GRP:
        file = files[0]
        filename = '{:02}'.format(i)  # Save the results by the subjects number
        subject_id = i
    else:
        file = files[i]
        filename = utils.basename(file)
        subject_id = None

    print("Processing file {}: {}".format(i, filename))
    # Check for the presence of the current_idx file
    # If it's not present, then either no calculations have been done, or the final results have already been saved
    if os.path.isfile(
            os.path.join(
                raw_save_root, "Results/{}/TE/raw/{}_current_idx.txt".format(
                    save_folder, filename))):
        # Load previous results, which are always saved in the uncompressed format
        results = np.load(
            os.path.join(
                raw_save_root,
                "Results/{}/TE/raw/{}.npy".format(save_folder, filename)))
        p_values = np.load(
            os.path.join(
                raw_save_root, "Results/{}/TE/raw/p_values/{}_p.npy".format(
                    save_folder, filename)))
        dce = np.load(
            os.path.join(
                raw_save_root, "Results/{}/TE/raw/dce/{}_dce.npy".format(
                    save_folder, filename)))
        with open(
                os.path.join(
                    raw_save_root,
                    "Results/{}/TE/raw/{}_current_idx.txt".format(
                        save_folder, filename)), 'r') as f:
            idx_values = f.readline()
        idx_values = list(map(int, idx_values.split(',')))
        print("Loading previous results")
        print("Starting from index", idx_values)
    else:
        results, p_values, dce, idx_values = None, None, None, None
        # Check both compressed and uncompressed options. If this file exists but the current_idx file doesn't then the
        # final results have already been saved. Exit to avoid running again
        if glob.glob(
                os.path.join(
                    raw_save_root,
                    "Results/{}/TE/raw/p_values/{}_p.np*".format(
                        save_folder, filename))):
            print("Result already present")
            exit()

    # Load parameter file
    param_file = 'Results/{}/AIS/params/{}_params.csv'.format(
        save_folder, filename)
    if set_k_to_0:
        df = utils.loadData(file, get_params=False, subject_id=subject_id)
        param_df = pd.DataFrame(np.zeros((len(df), 2), dtype=int),
                                columns=['k', 'tau'])
    else:
        df, param_df = utils.loadData(file,
                                      get_params=True,
                                      param_file=param_file,
                                      subject_id=subject_id)

    data = utils.preprocess(df, **preprocessing_params)
    saver = TEResultSaver(filename, save_folder, raw_save_root)
    calc = startCalc(calc_type)

    # Do the calculations
    results, p_values, dce = getLocalsForAllRegionPairs(
        data,
        param_df,
        calc,
        compute_p,
        saver=saver,
        save_every=save_every,
        results=results,
        p_values=p_values,
        dce=dce,
        idx_values=idx_values,
        use_source_embedding=use_source_embedding)

    # Save the final results
    # Add back the trimmed sections at the start and end of the timeseries by padding with zeros
    padding = ((0, 0), (preprocessing_params.get('trim_start', 0),
                        preprocessing_params.get('trim_end', 0)))
    saver.save_final_result(results,
                            p_values,
                            dce,
                            padding=padding,
                            compress=compress)
    print("\nTime taken:", round((time.time() - start_time) / 60, 2), 'min')
# -*- coding: utf-8 -*-
"""
Created on Thu May 30 15:49:39 2013

@author: attale00
"""

import utils
import os

labelPath='/export/somalia/egger/aflw_all/quality/labels'
#sourcePath='/export/somalia/egger/aflw_all/montages'
sourcePath='/export/somalia/egger/aflw_all'
targetPath ='/local/attale00/AFLW_ALL/montage'
targetPath='/local/attale00/AFLW_ALL/original'

labelFiles=utils.getAllFiles(labelPath)
counter =0
for labelFile in labelFiles:
    f=open(labelPath+'/'+labelFile)
    lines = f.readlines()
    
    if lines[1].startswith('fit=good') or lines[4].startswith('fit_mouth=good'):
        counter +=1
        os.symlink(sourcePath+'/'+labelFile[:-13]+'.png',targetPath+'/'+labelFile[:-13]+'.png')
    
print counter
        
import sys
import plottingUtils
import classifierUtils
from sklearn import svm
from sklearn import linear_model
from sklearn.decomposition import FastICA  
from scipy import linalg
import matplotlib.pyplot as plt


path = '/local/attale00/AFLW_ALL/'
path_ea = '/local/attale00/AFLW_cropped/multiPIE_cropped3/'
path_label = '/local/attale00/a_labels'


allLabelFiles =  utils.getAllFiles(path_label)
    
labeledImages = [i[0:16]+'.png' for i in allLabelFiles]




def multiPiePose():



    path_pose = '/local/attale00/poseLabels/multipie'
    
    poseLabel = utils.parseLabelFiles(path_pose,'rotY',labeledImages,cutoffSeq='.png',suffix='.frog')
    
    poses = [(k,np.double(v)*180/np.pi) for k,v in poseLabel.iteritems()]
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 28 15:38:38 2013

@author: attale00
"""

import cv2
import numpy as np
import utils
path = '/local/attale00/GoodPose'
pathg = path+'/extracted_alpha/grayScale'

files = utils.getAllFiles(pathg)
cv2.namedWindow('thresh')
big = np.zeros((512,2*512),dtype='uint8')
for f in files:
    im = cv2.imread(pathg+'/'+f,-1)
    big[:,0:512]=im
    ret,imbw = cv2.threshold(im,155,255,cv2.THRESH_BINARY)
    big[:,512:1024]=imbw
    cv2.imshow('test',big)
    cv2.waitKey(0)

print('done')
def main(nJobs = 1):

    path = '/local/attale00'
    
    fileNames = utils.getAllFiles(path+'/targets');
    
    attribute = 'mouth'
    
    attribute_values = utils.parseLabelINIFile(path+'/mouth_labels/labels.ini',attribute);
    
    print('------------Attribute: \t'+attribute+' ---------------')
    for i in attribute_values:
        print('Value: \t'+i)
        
    print('----------------------------')
    print('----------parsing label files------')
    labs=utils.parseLabelFiles(path+'/mouth_labels','mouth',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    print('-----computing Features-----')
    #make 10 bin hist for each mouth
    #roi = (40,200,100,200)
    roi = (50,190,110,402) 
    roi2=(128,256,0,256)
    mouthSet = fg.dataContainer(labs)
    #fg.getHistogram(20,roi,hrange=(0,255),dataC = mouthSet,path = path+'/extracted/gradients/Direction/',ending='_0.png')
    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
    m=cv2.resize(np.uint8(eM),(256,256));
    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    dil = cv2.dilate(m,strel)
    
    
    m=dil>0;
#    em=m[roi[0]:roi[1],roi[2]:roi[3]]
#    m= m !=True
  
    fg.getHogFeature(mouthSet,roi2,path=path+'/targets/grayScaleSmall/',ending=None,extraMask = None)
    #fg.getPixelValues(mouthSet,roi,path=path_ea+'/',ending='_0.png',mask =m, scaleFactor = 10)    
    #fg.getColorHistogram(mouthSet,roi,path=path_ea+'/',ending='_0.png',colorspace=None,range=(1.0,255.0),bins = 20)   
    mouthSet.targetNum=map(utils.mapMouthLabels2Two,mouthSet.target)
    n_estimators = range(10,180,20);
    max_features = range(2,22,2)
    max_depth = range(5,40,5)
    max_depth.append(100)
    
    min_split = range(1,20,2)
    
    score=[]
    var = []
    for n in n_estimators:    
        scoresRF = _crossValidate(mouthSet, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet.data[0])),min_split = 5)
   
        score.append(scoresRF.mean())
        var.append(scoresRF.std())
        
    print scoresRF
    plt.errorbar(n_estimators,score,yerr=var)
    plt.xlabel('number of trees')
#    plt.ylabel('cross val score')
    
#    mouthSet2 = fg.dataContainer(labs)
#    roi=(256,512,0,512)
#    fg.getColorHistogram(mouthSet2,roi,path=path+'/targets/',ending=None,colorspace='lab',range=(1.,255.0),bins = 20)
#    mouthSet2.targetNum=map(utils.mapMouthLabels2Two,mouthSet2.target)
#    score=[]
#    var = []
#    for n in n_estimators:    
#        scoresRF = _crossValidate(mouthSet2, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet2.data[0])),min_split = 5)
#   
#        score.append(scoresRF.mean())
#        var.append(scoresRF.std())
#        
#    print scoresRF
#    plt.errorbar(n_estimators,score,yerr=var)
#    plt.xlabel('number of trees')
#    plt.ylabel('cross val score')
    
     

#    fg.getColorHistogram(mouthSet,roi,path=path_ea+'/',ending='_0.png',colorspace='lab',range=(100.0,255.0),bins = 20)
#
#    score=[]
#    var = []
#    for n in n_estimators:    
#        scoresRF = _crossValidate(mouthSet, max_depth = 20,n_estimators =n ,nJobs = nJobs,max_features = np.sqrt(len(mouthSet.data[0])),min_split = 5)
#   
#        score.append(scoresRF.mean())
#        var.append(scoresRF.std())
#        
#    print scoresRF
#    plt.errorbar(n_estimators,score,yerr=var)
#    plt.xlabel('number of trees')
#    plt.ylabel('cross val score')
#    plt.legend(['HOG','LAB','HOG+LAB'])
#    plt.title('20bins')
    
    plt.show()        
    
    #classifier
    #linSVM = svm.SVC(kernel = 'linear',C=1)
    
    #this takes forever: check if that can be true
    #scoresLinSVM = cross_validation.cross_val_score(linSVM,data,y=targetNum,n_jobs=-1,verbose = 1)
    
    #implement random forest classifier with verbosity level
#    roi_narrow=(60,160,130,382)
#    extraMask = np.load('/home/attale00/Desktop/emptyMouthMask.npy')
#    
#    fg.getMeanAndVariance(roi_narrow,mouthSet,path_ea+'/',extraMask = extraMask,ending='_0.png')
#    scoresRF = _crossValidate(mouthSet,max_features = 13)
#    print 'Orientation and mean and cov' +str(scoresRF)    
    return
def main(mode):
    path = '/local/attale00/AFLW_ALL/'
    path_ea = '/local/attale00/AFLW_cropped/cropped3/'
#    
    fileNames = utils.getAllFiles(path_ea);
#    minr = 10000;
#    for f in fileNames:
#        im = cv2.imread(path_ea+f,-1)
#        if im.shape[0]!=40 or im.shape[1]!=120:
#            print f
#            print im.shape
#        minr = minr if im.shape[0]>= minr else im.shape[0]
#    
#    print minr
#    
    
    labs=utils.parseLabelFiles(path+'/labels/labels','mouth_opening',fileNames,cutoffSeq='.png',suffix='_face0.labels')
    
    
    
    testSet = fg.dataContainer(labs)
    
    
    roi=(0,37,0,115)
    roi=None
    filters = None
    meanI = None    
    #roi=(44,84,88,168)    
    
    
#    eM=np.load('/home/attale00/Desktop/mouthMask.npy')
#    m=cv2.resize(np.uint8(eM),(256,256));
#    strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
#    dil = cv2.dilate(m,strel)
#    
#    m=dil>0;


    components = 150
 
    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(40,120),roi=roi)
#    X=fg.getAllImagesFlat(path_ea,testSet.fileNames,(120,40),roi=roi,resizeFactor = .5)
# 
# perform ICA
    if mode not in ['s','v']:
        ica = FastICA(n_components=components,whiten=True)
        ica.fit(X)
        meanI=np.mean(X,axis=0)
        X1=X-meanI
        data=ica.transform(X1)
        filters=ica.components_
        
    elif mode in ['s','v']:
        W=np.load('/home/attale00/Desktop/classifiers/patches/filterMP1.npy')
        m=np.load('/home/attale00/Desktop/classifiers/patches/meanIMP1.npy')
        X1=X-m
        data=np.dot(X1,W.T)    
    
    for i in range(len(fileNames)):
            testSet.data[i].extend(data[i,:])
    #orientations = 2

    #strel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
    #fg.getHogFeature(testSet,roi,path=path_ea,ending='.png',extraMask = None,orientations = 5, cells_per_block=(3,3),pixels_per_cell=(24,8),maskFromAlpha=False)
    #fg.getColorHistogram(testSet,(0,40,40,80),path=path_ea,ending='.png',colorspace='lab',bins=bins)
    #fg.getImagePatchStat(testSet,path=path_ea,patchSize=(4,12))
    #fg.getImagePatchStat(testSet,path='/local/attale00/AFLW_cropped/mouth_img_error/',patchSize=(4,12))
  
    #pca
#    n_samples, n_features = X.shape
#
#    mean_ = np.mean(X, axis=0)
#    X -= mean_
#    U, S, V = linalg.svd(X)
#    explained_variance_ = (S ** 2) / n_samples
#    explained_variance_ratio_ = (explained_variance_ /explained_variance_.sum())
#    K=V / S[:, np.newaxis] * np.sqrt(n_samples)
#    filters=K[:100]
#    data=np.dot(X,filters.T)    
    
   
            
    
    testSet.targetNum=map(utils.mapMouthLabels2Two,testSet.target)
    rf=classifierUtils.standardRF(max_features = 23,min_split=15,max_depth=70,n_estimators=150)
    #rf=classifierUtils.standardRF(max_features = 5,min_split=12,max_depth=45)
    #rf = svm.NuSVC()
    #rf = linear_model.SGDClassifier(loss='perceptron', eta0=1, learning_rate='constant', penalty=None)    
    if mode in ['s','v']:
        print 'Classifying with loaded classifier'
        _classifyWithOld(path,testSet,mode)
    elif mode in ['c']:
        print 'cross validation of data'
        rValues = classifierUtils.dissectedCV(rf,testSet)
        pickle.dump(rValues,open('patches_cv_ica_{}'.format(components),'w'))
    elif mode in ['save']:
        print 'saving new classifier'
        _saveRF(testSet,rf,filters=filters,meanI=meanI)
    else:
        print 'not doing anything'
Exemple #34
0
def getPopulationParameters(data_path, extension, calc, history_lengths,
                            delays, **preprocessing_params):
    """
    Finds the parameters for history length and delay by considering the AIS values which have been averaged over the population
    AIS is calculated for each subject, for each combination of parameters and then averaged across the subjects
    The history length and delay which gives the maximum averaged AIS is chosen as the population parameters

    Arguments:
        data_path -- Location of the data files
        extension -- File extension of the data
        calc -- The JIDT calculator
        history_lengths -- Range of possible history length values
        delays -- Range of possible delay values
        preprocessing_params -- Parameters passed to utils.preprocess for preprocessing the time series data.
                                Includes sampling_rate / sampling_interval, mean_processing_type, trim_start, trim_end,
                                fcutlow, fcuthigh, use_filtfilt
    
    Returns:
        parameters -- A numpy array with two columns, containing the (history_length, delay) of each region
    """
    print("Finding population parameters")
    files = utils.getAllFiles(data_path, extension)

    data_all = utils.loadData(files[0], return_all_subjects=True)

    if data_all.ndim == 3:
        # Check to see if it's the GRP data, which will be returned as a 3D array of (regions, time points, subjects)
        number_of_subjects = data_all.shape[2]
    else:
        number_of_subjects = len(files)

    regions = data_all.shape[0]
    # Initialise
    ais_values = np.empty(
        (regions, number_of_subjects, len(history_lengths), len(delays)))

    for i in range(number_of_subjects):
        if data_all.ndim == 3:
            data = data_all[:, :,
                            i]  # If it's the GRP data, get the data for each subject by slicing on the 3rd dimension
        elif i == 0:
            data = data_all
        else:  # If it's not the GRP data, just load the next file when the first one is done
            data = utils.loadData(files[i])

        data = utils.preprocess(data, **preprocessing_params)
        dces = None  # Just want to calculate the acls once for all the regions, so assign it as None first, and then replace it
        for region in range(regions):
            if dces is None:
                dces = utils.getDCE(data)
            ais_values[region, i] = runAllParameters(data[region], calc,
                                                     history_lengths, delays,
                                                     dces[region])

            # Print progress bar
            utils.update_progress(
                (i * regions + region) / (number_of_subjects * regions),
                end_text="  Subject {} of {}".format(i + 1,
                                                     number_of_subjects))

    parameters = np.zeros((regions, 2), dtype=int)
    mean_ais_values = ais_values.mean(axis=1)  # Average over the subjects

    for region in range(regions):
        max_idx = utils.getMaxIdx2D(
            mean_ais_values[region],
            print_=False)  # Find the argmax across the averaged AIS values
        history_length = history_lengths[max_idx[0]]
        delay = delays[max_idx[1]]
        parameters[region] = (history_length, delay)

    return parameters