def main():
    train_file ="/Users/phx/downloads/competetion/recipe/train.json"
    with open(train_file) as file:
        data = json.load(file)
    print("size of dataset %d" % len(data))

    data = preprocess(data)
    data = preprocess(data)

    train_data = [data[i] for i in xrange(0,len(data)) if i%3 !=0]
    test_data = [data[i] for i in xrange(0,len(data)) if i%3 ==0]

    #test_data= preprocess(test_data)

    attribute_map = getAttributeMap(train_data,1)

    print('attribute number : %d' % len(attribute_map))
    print(attribute_map)


    label_map = getLabelMap(data)
    print('label number : %d' %len(label_map))
    print(label_map)
    X,y = getDataSet(train_data,attribute_map,label_map)
    testX,testY= getDataSet(test_data,attribute_map,label_map)
    sgd = SGDClassifier(loss='log')
    generate_save_proba(sgd,X,y,testX,testY,"SGDClassifier.loss_log")
    mnb = MultinomialNB(alpha=0.08, class_prior=None, fit_prior=True)
    generate_save_proba(mnb,X,y,testX,testY,"MultinomialNB.alpha_0.08")
    rf = RandomForestClassifier(n_estimators=500)
    generate_save_proba(rf,X,y,testX,testY,"RandomForestClassifier.n_estimators_500")
    """
from preprocessor import preprocess, getDataSet, getLabelMap, getAttributeMap
import json

train_file = "/Users/phx/downloads/competetion/recipe/train.json"
with open(train_file) as file:
    data = json.load(file)
print("size of dataset %d" % len(data))

data = preprocess(data)

train_data = [data[i] for i in xrange(0, len(data)) if i % 3 != 0]
test_data = [data[i] for i in xrange(0, len(data)) if i % 3 == 0]

# test_data= preprocess(test_data)

attribute_map = getAttributeMap(train_data, 1)

print("attribute number : %d" % len(attribute_map))
print(attribute_map)


label_map = getLabelMap(data)
print("label number : %d" % len(label_map))
print(label_map)


def balanceData(data):
    categorized_data = {}
    for record in data:
        if not categorized_data.get(record["cuisine"]):
            categorized_data[record["cuisine"]] = [record]