vec_length=200,

doc_vec_length = 200,

min_count=20,

window=12,

window_r=12,

negative=20,

sg=0, # 0:cbow , 1:skipgram

cbow_type=1, # 0:average_concat +syn1_doc 1:average , 2:concatenate+syn1_doc , 3:concatenate , 4:average_concat

skip_gram_type=0, # 0:simple 1:average, 2:concatenate

alpha=0.025,

alpha_doc=0.025,

alpha_rate = 0.99,

alpha_flag=0, # 0:学習パラメータは減少させていく, 1:減少させない

cbow_mean=1, # 0:no 1:average

iteration = 10, # 学習回数

average_flag=0, # 0:word_vecの和のみ 1:doc_vecとword_vecの和

is_np_mean_syn1=0, #0: syn1の誤差を平均， 1:syn1の誤差を合計 (cbow_type=0,2のとき)

is_using_word2vec=0, # 1:word2vecの単語ベクトルを用いる, 0:用いない

is_using_wiki = 0, # 0:none 1:wiki

hs = 1, # 0:hsを使わない 1:使う

sample=1e-5, # 1e-5 頻度の高い単語を減らす

freeze_learn=0, # 0:更新する , 1: word2vecを使った場合に単語ベクトルを更新しない

random_learn_flag=0, # 1:ランダムに学習する, 0:与えられた文章から順に学習する

n_gram_mode = 0, # 0:変換しない 1,2,3-gram

null_vec_type = 0, # 0:zeros , 1: ones, 2:random

skip_concat = 0, # 0:not skip 1:nullをskip

sentences=None,

input_file=None

):

import unicodedata

# import uuid

import datetime

import zenhan

# uniq_id = uuid.uuid4()

d = datetime.datetime.today()

uniq_id = d.strftime("%Y-%m-%d_%H:%M:%S")

# 文字を正規化する

def clean_text(text):

# del_n = re.compile('\n')

# text = del_n.sub('',text)

text = text.lower()

text = unicodedata.normalize('NFKC', text)

text = zenhan.z2h(text,zenhan.ASCII|zenhan.DIGIT)

return text

import logging

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

print '...model setting'

model = paragraph_vector.ParagraphVector(size=vec_length, doc_vec_size=doc_vec_length, min_count=min_count, window=window,window_r=window_r, sg=sg, negative=negative, cbow_type=cbow_type,cbow_mean=cbow_mean,average_flag=average_flag,alpha_flag=alpha_flag,is_np_mean_syn1=is_np_mean_syn1,is_using_word2vec=is_using_word2vec,hs=hs,skip_gram_type=skip_gram_type,sample=sample,freeze_learn=freeze_learn,random_learn_flag=random_learn_flag,null_vec_type=null_vec_type,skip_concat=skip_concat)

print '..load input file & build vocab'

sentences_length = model.build_vocab(sentences=open(input_file))

if sentences_length > 0:

sentences_length += 1

alpha_doc_m = alpha_doc

alpha_m = alpha

for i in xrange(iteration):

print "**iteration : %d " % i

print "alpha %f" % alpha_doc_m

model.train(input_file=input_file, alpha=alpha_m, alpha_doc=alpha_doc_m,sentences_length=sentences_length)

save_filename = "trained_model_{}_iter{}.p".format(uniq_id,str(i))

# model.save(save_filename)

with open(save_filename, 'w') as output:

pickle.dump(model, output)

alpha_doc_m *= alpha_rate

alpha_m *= alpha_rate