def learnFeatureExistance(busImportantFeatures, userImportantFeatures, trainReviews, path):
    logger = logging.getLogger('signature.learnFE')
    logger.info('starting learnFeatureExistance from %d reviews'%len(trainReviews))
    fsw = featureStructureWorker()
    modelDict = dict()
    
    missed_prediction = dict()
    for f, feature in enumerate(fsw.featureIdicator):
        if not fsw.featureIdicator[feature]:
            continue
        logger.info('Start working with (%d) %s'%(f,feature))
        #get data
        X1, Y1, X2, Y2, missed = getFeatures(logger, feature, trainReviews, busImportantFeatures, userImportantFeatures)
        missed_prediction[feature] = [missed, len(Y1)]
        
        
#        stat_line = '%d (%d/%d)reviews (%d of them pos(%d)/neg(%d))'%(len(Y1),sum(Y1),len(Y1) - sum(Y1),
#                                                                      len(Y2),sum(Y2),len(Y2) - sum(Y2))

        logger.debug('Got features for %d (%d/%d)reviews (%d of them pos(%d)/neg(%d))'%(len(Y1),sum(Y1),len(Y1) - sum(Y1),
                                                                                     len(Y2),sum(Y2),len(Y2) - sum(Y2)))

        print(len(Y1),len(Y2))
        if len(Y1) < 100 or sum(Y1) < 50 or len(Y1) - sum(Y1) < 50:
            continue
        if len(Y2) < 100 or sum(Y2) < 50 or len(Y2) - sum(Y2) < 50:
            continue

#        if len(Y1) < 10 or sum(Y1) < 10 or len(Y1) - sum(Y1) < 10:
#            continue
#        if len(Y2) < 10 or sum(Y2) < 10 or len(Y2) - sum(Y2) < 10:
#            continue


#        #cross validation
#        indicator = range(len(X))
#        random.shuffle(indicator)
#        thres = int(len(indicator)*0.8)
#        trainX = np.array([X[i] for i in indicator[:thres]])
#        trainY = np.array([Y[i] for i in indicator[:thres]])
#        testX = np.array([X[i] for i in indicator[thres:]])
#        testY = np.array([Y[i] for i in indicator[thres:]])
        
        #Logistic Regression
        bestThres, bestQ,logmodel = getLogModel(logger, feature, X1, Y1, path)
        
        logger.info('Sentiment prediction for (%d) %s'%(f,feature))
        #Logistic Regression
        bestThres_2, bestQ_2, logmodel_2 = getLogModel(logger, feature, X2, Y2, path)
        
        
        feat_info = [len(Y1), sum(Y1), len(Y1) - sum(Y1)] + bestQ + [len(Y2), sum(Y2),len(Y2) - sum(Y2)] + bestQ_2
        
        #bestThresSVM,bestF1SVM,svmmodel = getBestSVMModel(logger, feature, X, Y, path)
        
#       crossValidation(logger, np.array(X), np.array(Y))
        
        
        modelDict[feature] = [bestThres, logmodel, bestThres_2, logmodel_2, feat_info]
        
#        print(f)
#        if f > 6:
#            break
        
    return modelDict
def applyFeatureExistance(busImportantFeatures, userImportantFeatures, testReviews, modelDict, path):
    logger = logging.getLogger('signature.applyFE.aFE')
    logger.info('starting applyFeatureExistance from %d reviews'%len(testReviews))
    fsw = featureStructureWorker()
    featureWeights = dict()
    featureSWeights = dict()
    
    featureQuality = dict()
    
    for k, feature in enumerate(fsw.featureIdicator):
#        print(k,feature)
#        if k > 15:
#            break
        
        
        if not fsw.featureIdicator[feature]:
            continue
        if feature not in modelDict:
            continue
        
        logger.debug('Start working with (%d) %s'%(k,feature))
        #get data
        X1, Y1, X2, Y2, missed = getFeatures(logger, feature, testReviews, busImportantFeatures, userImportantFeatures)
        
        
        #weight = frequency
        featureWeights[feature] = float(sum(Y1))/len(Y1)
        #weight = sentiment
        featureSWeights[feature] = float(sum(Y2))/len(Y2)
        
        '''
        Existence
        '''
        #Ypred = [int(x[1] > modelDict[feature][0])  for x in modelDict[feature][1].predict_proba(np.array(X1))]
        Ypred = modelDict[feature][1].predict(np.array(X1))
        Yreal = np.array(Y1)
        
        quality = list(f1_score(Yreal, Ypred, average=None)) 
        quality += list(precision_score(Yreal, Ypred, average=None)) 
        quality += list(recall_score(Yreal, Ypred, average=None))
        
        '''
        Sentiment
        '''
        #YSpred = [int(x[1] > modelDict[feature][2]) for x in modelDict[feature][3].predict_proba(np.array(X2))]
        YSpred = modelDict[feature][3].predict(np.array(X2))
        YSreal = np.array(Y2)
        
        qualityS = list(f1_score(YSreal, YSpred, average=None)) 
        qualityS += list(precision_score(YSreal, YSpred, average=None)) 
        qualityS += list(recall_score(YSreal, YSpred, average=None))
        
        featureQuality[feature]  = [round(featureWeights[feature],2), len(Y1)]
        featureQuality[feature] += [round(x,2) for x in quality]
        featureQuality[feature] += [round(featureSWeights[feature],2), len(Y2)]
        featureQuality[feature] += [round(x,2) for x in qualityS]
        
#        print(feature,featureQuality[feature])
        
        for r, review in enumerate(testReviews):
            existence = 0
            predictedExistence = 0
            
            X1, Y1, X2, Y2, missed = getFeatures(logger, feature, [review], busImportantFeatures, userImportantFeatures)
            if len(Y1): #check if the review has enough history
                review['exPredFeatures'] = review.get('exPredFeatures', {})
            
                existence = Y1[0]
                #print Yreal[r], Ypred[r], modelDict[feature][0]
                
                
                prediction = modelDict[feature][1].predict_proba(np.array(X1))[0][1] # probability of second class!!!
                #prediction = float(modelDict[feature][1].predict(np.array(X1))[0])
                #prediction = busImportantFeatures[review['business_id']]['featureFreq'][feature]/100.0
                if prediction >= modelDict[feature][0]:
                    predictedExistence = 1
                else:
                    predictedExistence = 0
                predictedExistence = prediction
#                print(X1[0], prediction, busImportantFeatures[review['business_id']]['featureFreq'][feature]/100.0)
                randomPrediction = random.random()#int(random.random() > 0.5)
                simplePrediction = busImportantFeatures[review['business_id']]['featureFreq'][feature]/100.0#int(busImportantFeatures[review['business_id']]['featureFreq'][feature] > 40)
                basePredictionPos = 1
                basePredictionNeg = 0
                
                #print(existence, predictedExistence, randomPrediction, simplePrediction, basePredictionPos, basePredictionNeg)
                
                
                review['exPredFeatures'][feature] = [existence, predictedExistence,
                                                     randomPrediction, simplePrediction, 
                                                     basePredictionPos, basePredictionNeg]
                    
                #print(feature, review['exPredFeatures'][feature])
            
            '''
            Sentiment
            '''
            if len(Y2):
                review['sentPredFeatures'] = review.get('sentPredFeatures', {})
            
                sentiment = Y2[0]
                #print Yreal[r], Ypred[r], modelDict[feature][0]
                
                prediction = modelDict[feature][3].predict_proba(np.array(X2))[0][1]
                #prediction = float(modelDict[feature][3].predict(np.array(X2))[0])
                if prediction >= modelDict[feature][2]:
                    predictedSentiment = 1
                else:
                    predictedSentiment = 0
                predictedSentiment = prediction
                
                randomSPrediction = random.random()#int(random.random() > 0.5)
                simpleSPrediction = (busImportantFeatures[review['business_id']]['sentiment'].get(feature,[0.0,0])[0]+1)/2.0#int(busImportantFeatures[review['business_id']]['sentiment'].get(feature,[0.0,0])[0] >= -0.5)
                baseSPredictionPos = 1
                baseSPredictionNeg = 0
                
                review['sentPredFeatures'][feature] = [sentiment, predictedSentiment,
                                                       randomSPrediction, simpleSPrediction,
                                                       baseSPredictionPos, baseSPredictionNeg]
            
            if not r%5000:
                logger.debug('%d reviews processed'%r)
    
    return testReviews, featureWeights, featureQuality