def main():
    data_directory = "../../paper/data/dianping/corpus/"
    vector_directory = "../../paper/data/dianping/w2v/vector/"
    user_data = os.path.join(data_directory, "comment.keyword.train.user.1000")
    user_vector = os.path.join(vector_directory, "comment.keyword.train.user.vector")
    shop_data = os.path.join(data_directory, "comment.keyword.train.shop.1000")
    shop_vector = os.path.join(vector_directory, "comment.keyword.train.shop.vector")
    tool.logging("generating users'vector")
    get_vec(user_data, user_vector)
    tool.logging("generating shops'vector")
    get_vec(shop_data, shop_vector)
def get_vec(file_data, file_write):
    fw = open(file_write, 'w')
    tool.logging("traning %s data" % file_data)
    starttime = datetime.now()
    index = 0
    with open(file_data) as f:
        for line in f:
            index += 1
            if index % 500 ==0:
                tool.logging("%d cases" % index)
            arr = line.strip().split("\t")
            if len(arr) < 2:continue
            sen_vec = w2v_vector(" ".join(arr[1:]))
            line_w = arr[0] + '\t' + '\t'.join([str(x) for x in sen_vec]) + "\n"
            fw.write(line_w)
    fw.close()
    tool.logging("training %s data, eplased time:%s" % (file_data, str(datetime.now() - starttime)))
#coding = utf-8
import sys
sys.path.append("../langconv/")
from langconv import *
from datetime import datetime
import os
import jieba
import tool

stop_words = [word.strip() for word in open("../../paper/data/dianping/stopwords.txt").readlines()]
w2v_directory = "../../paper/data/dianping/w2v/model/"
word2vec_file = os.path.join(w2v_directory, "vec.txt")
#word2vec_file = "word2vec.txt"
tool.logging("loading word2vec_model")
word2vec = tool.load_vector_file(word2vec_file)
dim = open(word2vec_file).readline().strip().split()[1]

def w2v_vector(sentence):
    sentence = Converter('zh-hans').convert(sentence.decode("utf-8")).encode("utf-8")
    sen_vec = [0.0] * int(dim)
    count = 0
    for word in jieba.cut(sentence):
        if word2vec.has_key(word.lower().encode("utf-8")):
            sen_vec = tool.vec_add(sen_vec, word2vec[word.lower().encode("utf-8")])
            count += 1
    if count != 0:sen_vec = [x * 1.0 / int(count) for x in sen_vec]
    return sen_vec

def get_vec(file_data, file_write):
    fw = open(file_write, 'w')
    tool.logging("traning %s data" % file_data)