def cons_train_sample_for_cla(filename,indexs,local_fun,dic_path,sample_save_path,delete,str_splitTag,tc_splitTag):
'''根据提供的词典，将指定文件中的指定位置上的内容构造成SVM所需的问题格式，并进行保存'''
dic_list,global_weight = fileutil.read_dic_ex(dic_path,dtype=str)
if type(local_fun)==types.StringType:
local_fun = measure.local_f(local_fun)
label = set()
#对原训练样本进行词干化处理
print "-----------------正在对源文本进行词干化处理-------------------"
stem.stemFile(filename,str_splitTag,tc_splitTag)
f= file(filename,'r')
fs = file(sample_save_path,'w')
for line in f.readlines():
text = line.strip().split(tc_splitTag)
text_temp=""
if len(text)<indexs[len(indexs)-1]+1:
continue
for i in indexs:
text_temp+=str_splitTag+text[i]
y,x = ctmutil.cons_pro_for_svm(text[0],text_temp.strip().split(str_splitTag),dic_list,local_fun,global_weight)
if delete == True and len(x[0])==0:
continue
save_dic_train_sample(fs,y,x)
label.add(y[0])
f.close()
fs.close()
return label
def to_svm(tids, global_weight_dic, local_fun, class_id=None):
""" 根据词典和权重词典构造svm分类所需的输入格式
:param tids:
:param tok_dic: 词典
:param class2id: 类映射
:param global_weight_dic:权重词典
:return: feat
"""
local_fun = measure.local_f(local_fun)
feat = {}
#buidl feature vector
for tid in tids:
if tid in global_weight_dic:
if tid in feat:
feat[tid] += 1.0
else:
feat[tid] = 1.0
#compute feature weight
for tid, weight in feat.items():
feat[tid] = 1.0 * local_fun(weight) * global_weight_dic[tid]
#normalize
vec_sum = sum([weight**2.0 for weight in feat.values()])
vec_length = math.sqrt(vec_sum)
if vec_length != 0:
for tok, weight in feat.items():
feat[tok] = 1.0 * weight / vec_length
if class_id is not None:
return feat, class_id
else:
return feat
def cons_train_sample_for_cla(filename, indexs, local_fun, dic_path,
sample_save_path, delete, str_splitTag,
tc_splitTag):
'''根据提供的词典，将指定文件中的指定位置上的内容构造成SVM所需的问题格式，并进行保存'''
dic_list, global_weight = fileutil.read_dic_ex(dic_path, dtype=str)
if type(local_fun) == types.StringType:
local_fun = measure.local_f(local_fun)
label = set()
#对原训练样本进行词干化处理
print "-----------------正在对源文本进行词干化处理-------------------"
stem.stemFile(filename, str_splitTag, tc_splitTag)
f = file(filename, 'r')
fs = file(sample_save_path, 'w')
for line in f.readlines():
text = line.strip().split(tc_splitTag)
text_temp = ""
if len(text) < indexs[len(indexs) - 1] + 1:
continue
for i in indexs:
text_temp += str_splitTag + text[i]
y, x = ctmutil.cons_pro_for_svm(text[0],
text_temp.strip().split(str_splitTag),
dic_list, local_fun, global_weight)
if delete == True and len(x[0]) == 0:
continue
save_dic_train_sample(fs, y, x)
label.add(y[0])
f.close()
fs.close()
return label
def to_svm(tids, global_weight_dic, local_fun, class_id=None):
""" 根据词典和权重词典构造svm分类所需的输入格式
:param tids:
:param tok_dic: 词典
:param class2id: 类映射
:param global_weight_dic:权重词典
:return: feat
"""
local_fun = measure.local_f(local_fun)
feat = {}
#buidl feature vector
for tid in tids:
if tid in global_weight_dic:
if tid in feat:
feat[tid] += 1.0
else:
feat[tid] = 1.0
#compute feature weight
for tid,weight in feat.items():
feat[tid] = 1.0 * local_fun(weight) * global_weight_dic[tid]
#normalize
vec_sum = sum([weight**2.0 for weight in feat.values()])
vec_length = math.sqrt(vec_sum)
if vec_length!=0:
for tok, weight in feat.items():
feat[tok] = 1.0*weight/vec_length
if class_id is not None:
return feat,class_id
else:
return feat
def load_conf(model_dir, conf_file):
f = file(os.path.join(model_dir, conf_file), 'r')
for line in f.readlines():
text = line.split(":")
if text[0].strip() == "LocalFun":
local_fun = measure.local_f(text[1].strip())
return local_fun
def cal_sc_optim(lab,m,text,dic_list,local_fun,global_weight,str_splitTag):
'''输入标签，模型，待预测的文本，词典，以及词分词用的符号
返回的是一个预测标签与得分
'''
local_fun = measure.local_f(local_fun)
y,x = cons_pro_for_svm(lab,text.strip().split(str_splitTag),dic_list,local_fun,global_weight)
p_lab,p_acc,p_sc=tms_svm.predict(y,x,m)
return p_lab[0],tms_svm.classer_value(p_sc[0])
def cal_sc_optim(lab,m,text,dic_list,local_fun,global_weight,str_splitTag):
'''输入标签，模型，待预测的文本，词典，以及词分词用的符号
返回的是一个预测标签与得分，如果是二分类，返回的是直接得分，如果为多分类，返回的是经过计算的综合分数。
'''
local_fun = measure.local_f(local_fun)
y,x = ctmutil.cons_pro_for_svm(lab,text.strip().split(str_splitTag),dic_list,local_fun,global_weight)
p_lab,p_acc,p_sc=tms_svm.predict(y,x,m)
#在这里要判定是二分类还是多分类，如果为二分类，返回相应的分数，如果为多分类，则返回预测的标签。
return p_lab[0],tms_svm.classer_value(p_sc[0])
def train(train_docs, main_save_path,
config_name, model_name, train_name, param_name, svm_param, ratio, delete,
param_select, global_fun, local_fun):
'''
训练的自动化程序，分词,先进行特征选择，重新定义词典，根据新的词典，自动选择SVM最优的参数。
然后使用最优的参数进行SVM分类，最后生成训练后的模型。
需要保存的文件：（需定义一个主保存路径）
模型文件：词典.key+模型.model
临时文件 ：svm分类数据文件.train
'''
print "-----------------创建模型文件保存的路径-----------------"
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path, "model")) is False:
os.makedirs(os.path.join(main_save_path, "model"))
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path, "temp")) is False:
os.makedirs(os.path.join(main_save_path, "temp"))
#读取停用词文件
if stopword_filename == "":
stop_words_dic = dict()
else:
stop_words_dic = utils.read_dic(stopword_filename)
print "-----------------现在正在进行特征选择---------------"
dic_path = os.path.join(main_save_path, "model", "dic.key")
feature_select.feature_select(train_docs, global_fun, dic_path, ratio, stop_words_dic)
print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- "
problem_save_path = os.path.join(main_save_path, "temp", train_name)
label = cons_train_sample_for_cla(train_docs, measure.local_f(local_fun), dic_path, problem_save_path, delete)
print"--------------------选择最优的c,g------------------------------"
if param_select is True:
search_result_save_path = os.path.join(main_save_path, "temp", param_name)
coarse_c_range = (-5, 7, 2)
coarse_g_range = (1, 1, 1)
fine_c_step = 0.5
fine_g_step = 0
c, g = grid_search_param.grid(problem_save_path, search_result_save_path, coarse_c_range,
coarse_g_range, fine_c_step, fine_g_step)
svm_param = " -c " + str(c)
print "-----------------训练模型，并将模型进行保存----------"
model_save_path = os.path.join(main_save_path, "model", model_name)
ctm_train_model(problem_save_path, svm_param, model_save_path)
print "-----------------保存模型配置-----------------"
f_config = file(os.path.join(main_save_path, "model", config_name), 'w')
save_config(f_config, model_name, local_fun, global_fun, svm_param, label)
f_config.close()
print "-----------------训练结束---------------------"
def cal_sc_optim(lab, m, text, dic_list, local_fun, global_weight,
str_splitTag):
'''输入标签，模型，待预测的文本，词典，以及词分词用的符号
返回的是一个预测标签与得分
'''
local_fun = measure.local_f(local_fun)
y, x = cons_pro_for_svm(lab,
text.strip().split(str_splitTag), dic_list,
local_fun, global_weight)
p_lab, p_acc, p_sc = tms_svm.predict(y, x, m)
return p_lab[0], tms_svm.classer_value(p_sc[0])
def load_tms_model(config_file):
'''通过模型配置文件加载词典、全局因子、局部因子、SVM模型'''
model_main_path = os.path.dirname(config_file)
f = file(config_file,'r')
for line in f.readlines():
text = line.split(":")
if text[0].strip()=="DicName":
dic,global_weight = fileutil.read_dic_ex(os.path.join(model_main_path,text[1].strip()),dtype=str)
if text[0].strip()=="ModelName":
tms_svm.set_svm_type(tms_svm.detect_svm_type(os.path.join(model_main_path,text[1].strip())))
model= tms_svm.load_model(os.path.join(model_main_path,text[1].strip()))
if text[0].strip()=="LocalFun":
local_fun = measure.local_f(text[1].strip())
if text[0].strip()=="WordSeg":
seg = int(float(text[1]))
return local_fun,dic,global_weight,model,seg
def cons_train_sample_for_cla(train_docs, local_fun, dic_path, sample_save_path, delete):
'''根据提供的词典，将指定文件中的指定位置上的内容构造成SVM所需的问题格式，并进行保存'''
dic_list, global_weight = utils.read_dic_ex(dic_path, dtype=str)
local_fun = measure.local_f(local_fun)
label = set()
fs = file(sample_save_path, 'w')
for line in train_docs:
y, string = line.strip().split("\t")
x = utils.cons_pro_for_svm(string.strip().split(" "), dic_list, local_fun, global_weight)
y = [float(y)]
if delete is True and len(x[0]) == 0:
continue
save_dic_train_sample(fs, y, x)
label.add(y[0])
fs.close()
return label
def ctm_train(filename, indexes, main_save_path, stopword_filename, svm_param,
config_name, dic_name, model_name, train_name, svm_type,
param_name, ratio, delete, str_splitTag, tc_splitTag, seg,
param_select, global_fun, local_fun, label_file):
'''训练的自动化程序，分词,先进行特征选择，重新定义词典，根据新的词典，自动选择SVM最优的参数。
然后使用最优的参数进行SVM分类，最后生成训练后的模型。
需要保存的文件：（需定义一个主保存路径）
模型文件：词典.key+模型.model
临时文件 ：svm分类数据文件.train
filename 训练文本所在的文件名
indexs需要训练的指标项
main_save_path 模型保存的路径
stopword_filename 停用词的名称以及路径 ;默认不适用停用词
svm_type :svm类型：libsvm 或liblinear
svm_param 用户自己设定的svm的参数,这个要区分libsvm与liblinear参数的限制；例如"-s 0 -t 2 -c 0.2 "
dic_name 用户自定义词典名称;例如“dic.key”
model_name用户自定义模型名称 ;例如"svm.model"
train_name用户自定义训练样本名称 ；例如“svm.train”
param_name用户自定义参数文件名称 ；例如"svm.param"
ratio 特征选择保留词的比例 ；例如 0.4
delete对于所有特征值为0的样本是否删除,True or False
str_splitTag 分词所用的分割符号 例如"^"
tc_splitTag训练样本中各个字段分割所用的符号 ，例如"\t"
seg 分词的选择：0为不进行分词；1为使用mmseg分词；2为使用aliws分词
param_select ;是否进行SVM模型参数的搜索。True即为使用SVM模型grid.搜索，False即为不使用参数搜索。
local_fun：即对特征向量计算特征权重时需要设定的计算方式:x(i,j) = local(i,j)*global(i).可选的有tf,logtf
global_fun :全局权重的计算方式：有"one","idf","rf"
label_file:类标签的解释说明文件。
'''
print "-----------------创建模型文件保存的路径-----------------"
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path, "model")) is False:
os.makedirs(os.path.join(main_save_path, "model"))
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path, "temp")) is False:
os.makedirs(os.path.join(main_save_path, "temp"))
#设定SVM模型的类型。
tms_svm.set_svm_type(svm_type)
#如果没有给出停用词的文件名，则默认不使用停用词
if stopword_filename == "":
stop_words_dic = dict()
else:
stop_words_dic = fileutil.read_dic(stopword_filename)
#如果需要分词，则对原文件进行分词
if seg != 0:
print "-----------------正在对源文本进行分词-------------------"
segment_file = os.path.dirname(filename) + "/segmented"
segment.file_seg(filename, indexes, segment_file, str_splitTag,
tc_splitTag, seg)
filename = segment_file
#对原训练样本进行词干化处理
print "-----------------正在对源文本进行词干化处理-------------------"
stem.stemFile(filename, str_splitTag, tc_splitTag)
print "-----------------现在正在进行特征选择---------------"
dic_path = os.path.join(main_save_path, "model", dic_name)
feature_select(filename,
indexes,
global_fun,
dic_path,
ratio,
stop_words_dic,
str_splitTag=str_splitTag,
tc_splitTag=tc_splitTag)
print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- "
problem_save_path = os.path.join(main_save_path, "temp", train_name)
local_fun_str = local_fun
local_fun = measure.local_f(local_fun)
label = cons_train_sample_for_cla(filename, indexes, local_fun, dic_path,
problem_save_path, delete, str_splitTag,
tc_splitTag)
if param_select == True:
print "--------------------选择最优的c,g------------------------------"
search_result_save_path = main_save_path + "temp/" + param_name
if svm_type == "libsvm":
coarse_c_range = (-5, 7, 2)
coarse_g_range = (3, -10, -2)
fine_c_step = 0.5
fine_g_step = 0.5
c, g = grid_search_param.grid(problem_save_path,
search_result_save_path, svm_type,
coarse_c_range, coarse_g_range,
fine_c_step, fine_g_step)
svm_param = svm_param + " -c " + str(c) + " -g " + str(g)
if svm_type == "liblinear" or (svm_type == "libsvm" and
is_linear_kernal(svm_param) is True):
coarse_c_range = (-5, 7, 2)
coarse_g_range = (1, 1, 1)
fine_c_step = 0.5
fine_g_step = 0
c, g = grid_search_param.grid(problem_save_path,
search_result_save_path, svm_type,
coarse_c_range, coarse_g_range,
fine_c_step, fine_g_step)
svm_param = svm_param + " -c " + str(c)
print "-----------------训练模型，并将模型进行保存----------"
model_save_path = main_save_path + "model/" + model_name
ctm_train_model(problem_save_path, svm_type, svm_param, model_save_path)
print "-----------------保存模型配置-----------------"
f_config = file(os.path.join(main_save_path, "model", config_name), 'w')
save_config(f_config, dic_name, model_name, local_fun_str, global_fun, seg,
svm_type, svm_param, label_file, label)
f_config.close()
def ctm_train(filename,indexes,main_save_path,stopword_filename,svm_param,config_name,dic_name,model_name,train_name,svm_type,param_name,ratio,delete,str_splitTag,tc_splitTag,seg,param_select,global_fun,local_fun,label_file):
'''训练的自动化程序，分词,先进行特征选择，重新定义词典，根据新的词典，自动选择SVM最优的参数。
然后使用最优的参数进行SVM分类，最后生成训练后的模型。
需要保存的文件：（需定义一个主保存路径）
模型文件：词典.key+模型.model
临时文件 ：svm分类数据文件.train
filename 训练文本所在的文件名
indexs需要训练的指标项
main_save_path 模型保存的路径
stopword_filename 停用词的名称以及路径 ;默认不适用停用词
svm_type :svm类型：libsvm 或liblinear
svm_param 用户自己设定的svm的参数,这个要区分libsvm与liblinear参数的限制；例如"-s 0 -t 2 -c 0.2 "
dic_name 用户自定义词典名称;例如“dic.key”
model_name用户自定义模型名称 ;例如"svm.model"
train_name用户自定义训练样本名称 ；例如“svm.train”
param_name用户自定义参数文件名称 ；例如"svm.param"
ratio 特征选择保留词的比例 ；例如 0.4
delete对于所有特征值为0的样本是否删除,True or False
str_splitTag 分词所用的分割符号 例如"^"
tc_splitTag训练样本中各个字段分割所用的符号 ，例如"\t"
seg 分词的选择：0为不进行分词；1为使用mmseg分词；2为使用aliws分词
param_select ;是否进行SVM模型参数的搜索。True即为使用SVM模型grid.搜索，False即为不使用参数搜索。
local_fun：即对特征向量计算特征权重时需要设定的计算方式:x(i,j) = local(i,j)*global(i).可选的有tf,logtf
global_fun :全局权重的计算方式：有"one","idf","rf"
label_file:类标签的解释说明文件。
'''
print "-----------------创建模型文件保存的路径-----------------"
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path,"model")) is False:
os.makedirs(os.path.join(main_save_path,"model"))
if os.path.exists(main_save_path):
if os.path.exists(os.path.join(main_save_path,"temp")) is False:
os.makedirs(os.path.join(main_save_path,"temp"))
#设定SVM模型的类型。
tms_svm.set_svm_type(svm_type)
#如果没有给出停用词的文件名，则默认不使用停用词
if stopword_filename =="":
stop_words_dic=dict()
else:
stop_words_dic = fileutil.read_dic(stopword_filename)
#如果需要分词，则对原文件进行分词
if seg!=0:
print "-----------------正在对源文本进行分词-------------------"
segment_file = os.path.dirname(filename)+"/segmented"
segment.file_seg(filename,indexes,segment_file,str_splitTag,tc_splitTag,seg)
filename = segment_file
#对原训练样本进行词干化处理
print "-----------------正在对源文本进行词干化处理-------------------"
stem.stemFile(filename,str_splitTag,tc_splitTag)
print "-----------------现在正在进行特征选择---------------"
dic_path= os.path.join(main_save_path,"model",dic_name)
feature_select(filename,indexes,global_fun,dic_path,ratio,stop_words_dic,str_splitTag=str_splitTag,tc_splitTag=tc_splitTag)
print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- "
problem_save_path = os.path.join(main_save_path,"temp",train_name)
local_fun_str = local_fun
local_fun = measure.local_f(local_fun)
label = cons_train_sample_for_cla(filename,indexes,local_fun,dic_path,problem_save_path,delete,str_splitTag,tc_splitTag)
if param_select ==True:
print"--------------------选择最优的c,g------------------------------"
search_result_save_path = main_save_path +"temp/"+param_name
if svm_type=="libsvm":
coarse_c_range=(-5,7,2)
coarse_g_range=(3,-10,-2)
fine_c_step=0.5
fine_g_step=0.5
c,g=grid_search_param.grid(problem_save_path,search_result_save_path,svm_type,coarse_c_range,coarse_g_range,fine_c_step,fine_g_step)
svm_param = svm_param + " -c "+str(c)+" -g "+str(g)
if svm_type=="liblinear" or (svm_type=="libsvm" and is_linear_kernal(svm_param) is True):
coarse_c_range=(-5,7,2)
coarse_g_range=(1,1,1)
fine_c_step=0.5
fine_g_step=0
c,g=grid_search_param.grid(problem_save_path,search_result_save_path,svm_type,coarse_c_range,coarse_g_range,fine_c_step,fine_g_step)
svm_param = svm_param + " -c "+str(c)
print "-----------------训练模型，并将模型进行保存----------"
model_save_path = main_save_path+"model/"+model_name
ctm_train_model(problem_save_path,svm_type,svm_param,model_save_path)
print "-----------------保存模型配置-----------------"
f_config = file(os.path.join(main_save_path,"model",config_name),'w')
save_config(f_config,dic_name,model_name,local_fun_str,global_fun,seg,svm_type,svm_param,label_file,label)
f_config.close()
