def tms_grid_param(problem_path, result_save_path="../svm.param", svm_type="libsvm", coarse_c_range=(-5, 7, 2), coarse_g_range=(3, -10, -2), fine_c_step=0.5, fine_g_step=0.5): '''对SVM的参数进行搜索,如果是libsvm则搜索(c,gamma),如果是liblinear则搜索(c)。当训练样本的容量大于3000时就会在粗粒度搜索时使用子集,子集的大小为[3000,5000]范围内。 当 必须参数: problem_path:SVM输入格式文件的路径即名称。 可选参数: result_save_path:结果文件的保存路径:默认为"../svm.param" svm_type :选择的SVM的类型,默认为libsvm coarse_c_range :粗粒度搜索时c搜索的范围,默认情况下为[-5,7],步长为2 coarse_g_range:粗粒度搜索时g搜索的范围,默认情况下为[3,-10]步长为-2 fine_c_step :细粒度搜索时c的步长,默认情况下为0.5 fine_c_step :细粒度搜索时c的步长,默认情况下为0.5 ''' if svm_type == "liblinear": coarse_g_range = (1, 1, 1) fine_g_step = 0 c, g = grid_search_param.grid(problem_path, result_save_path, svm_type, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) print "best c=%s ,g=%s" % (c, g) return c, g
def train(train_docs, main_save_path, config_name, model_name, train_name, param_name, svm_param, ratio, delete, param_select, global_fun, local_fun): ''' 训练的自动化程序,分词,先进行特征选择,重新定义词典,根据新的词典,自动选择SVM最优的参数。 然后使用最优的参数进行SVM分类,最后生成训练后的模型。 需要保存的文件:(需定义一个主保存路径) 模型文件:词典.key+模型.model 临时文件 :svm分类数据文件.train ''' print "-----------------创建模型文件保存的路径-----------------" if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path, "model")) is False: os.makedirs(os.path.join(main_save_path, "model")) if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path, "temp")) is False: os.makedirs(os.path.join(main_save_path, "temp")) #读取停用词文件 if stopword_filename == "": stop_words_dic = dict() else: stop_words_dic = utils.read_dic(stopword_filename) print "-----------------现在正在进行特征选择---------------" dic_path = os.path.join(main_save_path, "model", "dic.key") feature_select.feature_select(train_docs, global_fun, dic_path, ratio, stop_words_dic) print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- " problem_save_path = os.path.join(main_save_path, "temp", train_name) label = cons_train_sample_for_cla(train_docs, measure.local_f(local_fun), dic_path, problem_save_path, delete) print"--------------------选择最优的c,g------------------------------" if param_select is True: search_result_save_path = os.path.join(main_save_path, "temp", param_name) coarse_c_range = (-5, 7, 2) coarse_g_range = (1, 1, 1) fine_c_step = 0.5 fine_g_step = 0 c, g = grid_search_param.grid(problem_save_path, search_result_save_path, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) svm_param = " -c " + str(c) print "-----------------训练模型,并将模型进行保存----------" model_save_path = os.path.join(main_save_path, "model", model_name) ctm_train_model(problem_save_path, svm_param, model_save_path) print "-----------------保存模型配置-----------------" f_config = file(os.path.join(main_save_path, "model", config_name), 'w') save_config(f_config, model_name, local_fun, global_fun, svm_param, label) f_config.close() print "-----------------训练结束---------------------"
def tms_grid_param(problem_path, result_save_path="../svm.param", svm_type="libsvm", coarse_c_range=(-5,7,2), coarse_g_range=(3,-10,-2), fine_c_step=0.5, fine_g_step=0.5): '''对SVM的参数进行搜索,如果是libsvm则搜索(c,gamma),如果是liblinear则搜索(c)。当训练样本的容量大于3000时就会在粗粒度搜索时使用子集,子集的大小为[3000,5000]范围内。 当 必须参数: problem_path:SVM输入格式文件的路径即名称。 可选参数: result_save_path:结果文件的保存路径:默认为"../svm.param" svm_type :选择的SVM的类型,默认为libsvm coarse_c_range :粗粒度搜索时c搜索的范围,默认情况下为[-5,7],步长为2 coarse_g_range:粗粒度搜索时g搜索的范围,默认情况下为[3,-10]步长为-2 fine_c_step :细粒度搜索时c的步长,默认情况下为0.5 fine_c_step :细粒度搜索时c的步长,默认情况下为0.5 ''' if svm_type=="liblinear": coarse_g_range=(1,1,1) fine_g_step=0 c,g = grid_search_param.grid(problem_path, result_save_path, svm_type, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) print "best c=%s ,g=%s"%(c,g) return c,g
def ctm_train(filename, indexes, main_save_path, stopword_filename, svm_param, config_name, dic_name, model_name, train_name, svm_type, param_name, ratio, delete, str_splitTag, tc_splitTag, seg, param_select, global_fun, local_fun, label_file): '''训练的自动化程序,分词,先进行特征选择,重新定义词典,根据新的词典,自动选择SVM最优的参数。 然后使用最优的参数进行SVM分类,最后生成训练后的模型。 需要保存的文件:(需定义一个主保存路径) 模型文件:词典.key+模型.model 临时文件 :svm分类数据文件.train filename 训练文本所在的文件名 indexs需要训练的指标项 main_save_path 模型保存的路径 stopword_filename 停用词的名称以及路径 ;默认不适用停用词 svm_type :svm类型:libsvm 或liblinear svm_param 用户自己设定的svm的参数,这个要区分libsvm与liblinear参数的限制;例如"-s 0 -t 2 -c 0.2 " dic_name 用户自定义词典名称;例如“dic.key” model_name用户自定义模型名称 ;例如"svm.model" train_name用户自定义训练样本名称 ;例如“svm.train” param_name用户自定义参数文件名称 ;例如"svm.param" ratio 特征选择保留词的比例 ;例如 0.4 delete对于所有特征值为0的样本是否删除,True or False str_splitTag 分词所用的分割符号 例如"^" tc_splitTag训练样本中各个字段分割所用的符号 ,例如"\t" seg 分词的选择:0为不进行分词;1为使用mmseg分词;2为使用aliws分词 param_select ;是否进行SVM模型参数的搜索。True即为使用SVM模型grid.搜索,False即为不使用参数搜索。 local_fun:即对特征向量计算特征权重时需要设定的计算方式:x(i,j) = local(i,j)*global(i).可选的有tf,logtf global_fun :全局权重的计算方式:有"one","idf","rf" label_file:类标签的解释说明文件。 ''' print "-----------------创建模型文件保存的路径-----------------" if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path, "model")) is False: os.makedirs(os.path.join(main_save_path, "model")) if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path, "temp")) is False: os.makedirs(os.path.join(main_save_path, "temp")) #设定SVM模型的类型。 tms_svm.set_svm_type(svm_type) #如果没有给出停用词的文件名,则默认不使用停用词 if stopword_filename == "": stop_words_dic = dict() else: stop_words_dic = fileutil.read_dic(stopword_filename) #如果需要分词,则对原文件进行分词 if seg != 0: print "-----------------正在对源文本进行分词-------------------" segment_file = os.path.dirname(filename) + "/segmented" segment.file_seg(filename, indexes, segment_file, str_splitTag, tc_splitTag, seg) filename = segment_file #对原训练样本进行词干化处理 print "-----------------正在对源文本进行词干化处理-------------------" stem.stemFile(filename, str_splitTag, tc_splitTag) print "-----------------现在正在进行特征选择---------------" dic_path = os.path.join(main_save_path, "model", dic_name) feature_select(filename, indexes, global_fun, dic_path, ratio, stop_words_dic, str_splitTag=str_splitTag, tc_splitTag=tc_splitTag) print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- " problem_save_path = os.path.join(main_save_path, "temp", train_name) local_fun_str = local_fun local_fun = measure.local_f(local_fun) label = cons_train_sample_for_cla(filename, indexes, local_fun, dic_path, problem_save_path, delete, str_splitTag, tc_splitTag) if param_select == True: print "--------------------选择最优的c,g------------------------------" search_result_save_path = main_save_path + "temp/" + param_name if svm_type == "libsvm": coarse_c_range = (-5, 7, 2) coarse_g_range = (3, -10, -2) fine_c_step = 0.5 fine_g_step = 0.5 c, g = grid_search_param.grid(problem_save_path, search_result_save_path, svm_type, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) svm_param = svm_param + " -c " + str(c) + " -g " + str(g) if svm_type == "liblinear" or (svm_type == "libsvm" and is_linear_kernal(svm_param) is True): coarse_c_range = (-5, 7, 2) coarse_g_range = (1, 1, 1) fine_c_step = 0.5 fine_g_step = 0 c, g = grid_search_param.grid(problem_save_path, search_result_save_path, svm_type, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) svm_param = svm_param + " -c " + str(c) print "-----------------训练模型,并将模型进行保存----------" model_save_path = main_save_path + "model/" + model_name ctm_train_model(problem_save_path, svm_type, svm_param, model_save_path) print "-----------------保存模型配置-----------------" f_config = file(os.path.join(main_save_path, "model", config_name), 'w') save_config(f_config, dic_name, model_name, local_fun_str, global_fun, seg, svm_type, svm_param, label_file, label) f_config.close()
def ctm_train(filename,indexes,main_save_path,stopword_filename,svm_param,config_name,dic_name,model_name,train_name,svm_type,param_name,ratio,delete,str_splitTag,tc_splitTag,seg,param_select,global_fun,local_fun,label_file): '''训练的自动化程序,分词,先进行特征选择,重新定义词典,根据新的词典,自动选择SVM最优的参数。 然后使用最优的参数进行SVM分类,最后生成训练后的模型。 需要保存的文件:(需定义一个主保存路径) 模型文件:词典.key+模型.model 临时文件 :svm分类数据文件.train filename 训练文本所在的文件名 indexs需要训练的指标项 main_save_path 模型保存的路径 stopword_filename 停用词的名称以及路径 ;默认不适用停用词 svm_type :svm类型:libsvm 或liblinear svm_param 用户自己设定的svm的参数,这个要区分libsvm与liblinear参数的限制;例如"-s 0 -t 2 -c 0.2 " dic_name 用户自定义词典名称;例如“dic.key” model_name用户自定义模型名称 ;例如"svm.model" train_name用户自定义训练样本名称 ;例如“svm.train” param_name用户自定义参数文件名称 ;例如"svm.param" ratio 特征选择保留词的比例 ;例如 0.4 delete对于所有特征值为0的样本是否删除,True or False str_splitTag 分词所用的分割符号 例如"^" tc_splitTag训练样本中各个字段分割所用的符号 ,例如"\t" seg 分词的选择:0为不进行分词;1为使用mmseg分词;2为使用aliws分词 param_select ;是否进行SVM模型参数的搜索。True即为使用SVM模型grid.搜索,False即为不使用参数搜索。 local_fun:即对特征向量计算特征权重时需要设定的计算方式:x(i,j) = local(i,j)*global(i).可选的有tf,logtf global_fun :全局权重的计算方式:有"one","idf","rf" label_file:类标签的解释说明文件。 ''' print "-----------------创建模型文件保存的路径-----------------" if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path,"model")) is False: os.makedirs(os.path.join(main_save_path,"model")) if os.path.exists(main_save_path): if os.path.exists(os.path.join(main_save_path,"temp")) is False: os.makedirs(os.path.join(main_save_path,"temp")) #设定SVM模型的类型。 tms_svm.set_svm_type(svm_type) #如果没有给出停用词的文件名,则默认不使用停用词 if stopword_filename =="": stop_words_dic=dict() else: stop_words_dic = fileutil.read_dic(stopword_filename) #如果需要分词,则对原文件进行分词 if seg!=0: print "-----------------正在对源文本进行分词-------------------" segment_file = os.path.dirname(filename)+"/segmented" segment.file_seg(filename,indexes,segment_file,str_splitTag,tc_splitTag,seg) filename = segment_file #对原训练样本进行词干化处理 print "-----------------正在对源文本进行词干化处理-------------------" stem.stemFile(filename,str_splitTag,tc_splitTag) print "-----------------现在正在进行特征选择---------------" dic_path= os.path.join(main_save_path,"model",dic_name) feature_select(filename,indexes,global_fun,dic_path,ratio,stop_words_dic,str_splitTag=str_splitTag,tc_splitTag=tc_splitTag) print "-----------------再根据特征选择后的词典构造新的SVM分类所需的训练样本------------------- " problem_save_path = os.path.join(main_save_path,"temp",train_name) local_fun_str = local_fun local_fun = measure.local_f(local_fun) label = cons_train_sample_for_cla(filename,indexes,local_fun,dic_path,problem_save_path,delete,str_splitTag,tc_splitTag) if param_select ==True: print"--------------------选择最优的c,g------------------------------" search_result_save_path = main_save_path +"temp/"+param_name if svm_type=="libsvm": coarse_c_range=(-5,7,2) coarse_g_range=(3,-10,-2) fine_c_step=0.5 fine_g_step=0.5 c,g=grid_search_param.grid(problem_save_path,search_result_save_path,svm_type,coarse_c_range,coarse_g_range,fine_c_step,fine_g_step) svm_param = svm_param + " -c "+str(c)+" -g "+str(g) if svm_type=="liblinear" or (svm_type=="libsvm" and is_linear_kernal(svm_param) is True): coarse_c_range=(-5,7,2) coarse_g_range=(1,1,1) fine_c_step=0.5 fine_g_step=0 c,g=grid_search_param.grid(problem_save_path,search_result_save_path,svm_type,coarse_c_range,coarse_g_range,fine_c_step,fine_g_step) svm_param = svm_param + " -c "+str(c) print "-----------------训练模型,并将模型进行保存----------" model_save_path = main_save_path+"model/"+model_name ctm_train_model(problem_save_path,svm_type,svm_param,model_save_path) print "-----------------保存模型配置-----------------" f_config = file(os.path.join(main_save_path,"model",config_name),'w') save_config(f_config,dic_name,model_name,local_fun_str,global_fun,seg,svm_type,svm_param,label_file,label) f_config.close()
def main(): usage = "usage:%prog [options] version=%prog 1.0" parser = OptionParser(usage=usage) parser.add_option("-s", "--step", type="choice", choices=["1", "2", "3", "4", "5"], dest="step", help="step1 is auto training the svm model") parser.add_option("-p", "--path", dest="save_main_path") parser.add_option("-P", "--problem_path", dest="problem_save_path") parser.add_option("-i", "--indexes", dest="indexes", action="callback", type="string", default=[1], callback=list_callback) parser.add_option("-w", "--stopword", action="store_false", dest="stopword", default=True) parser.add_option("-n", "--config_name", dest="config_name", default="tms.config") parser.add_option("-d", "--dic_name", dest="dic_name", default="dic.key") parser.add_option("-D", "--dic_path", dest="dic_path") parser.add_option("-m", "--model_name", dest="model_name", default="tms.model") parser.add_option("-t", "--train_name", dest="train_name", default="tms.train") parser.add_option("-a", "--param_name", dest="param_name", default="tms.param") parser.add_option("-r", "--ratio", dest="ratio", type="float", default=0.4) parser.add_option("-A", "--svm_param", dest="svm_param", default="'-s 0 -t 2 -c 1.0 -g 0.25'") parser.add_option("-T", "--tc_splitTag", dest="tc_splitTag", type="string", default="\t") parser.add_option("-S", "--str_splitTag", dest="str_splitTag", type="string", default="^") parser.add_option("-v", "--svm_type", dest="svm_type", default="libsvm", type="choice", choices=["libsvm", "liblinear"]) parser.add_option("-e", "--segment", type="choice", dest="segment", default=0, choices=[0, 1, 2]) parser.add_option("-c", "--param_select", action="store_false", dest="param_select", default=True) parser.add_option("-g", "--global_fun", dest="global_fun", default="one", type="choice", choices=["one", "idf", "rf"]) parser.add_option("-l", "--local_fun", dest="local_fun", default="tf", type="choice", choices=["tf"]) parser.add_option("-b", "--label_file", dest="label_file", type="string", default="") options, args = parser.parse_args() if options.indexes: indexes = [int(i) for i in options.indexes] if options.step: step = int(options.step) if options.stopword == False: stopword_filename = "" else: stopword_filename = os.path.dirname(args[0]) + "/stopwords.txt" if options.svm_param: svm_param = options.svm_param.replace("'", "") if step == 1: train_model.ctm_train(args[0], indexes, options.save_main_path, stopword_filename, config_name=options.config_name, svm_type=options.svm_type, segment=options.segment, param_select=options.param_select, global_fun=options.global_fun, local_fun=options.local_fun, svm_param=svm_param, dic_name=options.dic_name, model_name=options.model_name, train_name=options.train_name, param_name=options.param_name, ratio=options.ratio, delete=True, str_splitTag=options.str_splitTag, tc_splitTag=options.tc_splitTag, label_file=options.label_file) if step == 2: train_model.ctm_feature_select(args[0], indexes, options.global_fun, options.save_main_path, options.dic_name, options.ratio, stopword_filename, str_splitTag=options.str_splitTag, tc_splitTag=options.tc_splitTag) if step == 3: if os.path.exists(options.save_main_path): if os.path.exists(options.save_main_path + "temp/") is False: os.makedirs(options.save_main_path + "temp/") sample_save_path = options.save_main_path + "temp/svm.train" train_model.cons_train_sample_for_cla( args[0], indexes, options.local_fun, options.dic_path, sample_save_path, delete=True, str_splitTag=options.str_splitTag, tc_splitTag=options.tc_splitTag) if step == 4: search_result_save_path = options.save_main_path + "temp/" + "svm.param" tms_svm.set_svm_type(options.svm_type) if options.svm_type == "libsvm": coarse_c_range = (-5, 7, 2) coarse_g_range = (3, -10, -2) fine_c_step = 0.5 fine_g_step = 0.5 if options.svm_type == "liblinear": coarse_c_range = (-5, 7, 2) coarse_g_range = (1, 1, 1) fine_c_step = 0.5 fine_g_step = 0 c, g = grid_search_param.grid(args[0], search_result_save_path, options.svm_type, coarse_c_range, coarse_g_range, fine_c_step, fine_g_step) print "best c = %s\t g = %s\n" % (c, g) if step == 5: model_save_path = options.save_main_path + "model/" + options.model_name train_model.ctm_train_model(options.problem_save_path, svm_param, model_save_path)
def main(): usage ="usage:%prog [options] version=%prog 1.0" parser = OptionParser(usage=usage) parser.add_option("-s","--step",type="choice",choices=["1","2","3","4","5"],dest="step",help="step1 is auto training the svm model") parser.add_option("-p","--path",dest="save_main_path") parser.add_option("-P","--problem_path",dest="problem_save_path") parser.add_option("-i","--indexes",dest="indexes",action="callback",type="string",default=[1],callback=list_callback) parser.add_option("-w","--stopword",action="store_false",dest="stopword",default=True) parser.add_option("-n","--config_name",dest="config_name",default="tms.config") parser.add_option("-d","--dic_name",dest="dic_name",default="dic.key") parser.add_option("-D","--dic_path",dest="dic_path") parser.add_option("-m","--model_name",dest="model_name",default="tms.model") parser.add_option("-t","--train_name",dest="train_name",default="tms.train") parser.add_option("-a","--param_name",dest="param_name",default="tms.param") parser.add_option("-r","--ratio",dest="ratio",type="float",default=0.4) parser.add_option("-A","--svm_param",dest="svm_param",default="'-s 0 -t 2 -c 1.0 -g 0.25'") parser.add_option("-T","--tc_splitTag",dest="tc_splitTag",type="string",default="\t") parser.add_option("-S","--str_splitTag",dest="str_splitTag",type="string",default="^") parser.add_option("-v","--svm_type",dest="svm_type",default="libsvm",type="choice",choices=["libsvm","liblinear"]) parser.add_option("-e","--segment",type="choice",dest="segment",default=0,choices=[0,1,2]) parser.add_option("-c","--param_select",action="store_false",dest="param_select",default=True) parser.add_option("-g","--global_fun",dest="global_fun",default="one",type="choice",choices=["one","idf","rf"]) parser.add_option("-l","--local_fun",dest="local_fun",default="tf",type="choice",choices=["tf"]) parser.add_option("-b","--label_file",dest="label_file",type="string",default="") options, args = parser.parse_args() if options.indexes: indexes = [int(i) for i in options.indexes] if options.step: step = int(options.step) if options.stopword ==False: stopword_filename="" else: stopword_filename = os.path.dirname(args[0])+"/stopwords.txt" if options.svm_param: svm_param = options.svm_param.replace("'","") if step==1: train_model.ctm_train(args[0],indexes,options.save_main_path,stopword_filename,config_name=options.config_name,svm_type =options.svm_type,segment=options.segment,param_select=options.param_select,global_fun=options.global_fun,local_fun=options.local_fun,svm_param=svm_param,dic_name=options.dic_name,model_name=options.model_name,train_name=options.train_name,param_name=options.param_name,ratio=options.ratio,delete=True,str_splitTag=options.str_splitTag,tc_splitTag=options.tc_splitTag,label_file=options.label_file) if step==2: train_model.ctm_feature_select(args[0],indexes,options.global_fun,options.save_main_path,options.dic_name,options.ratio,stopword_filename,str_splitTag=options.str_splitTag,tc_splitTag=options.tc_splitTag) if step==3: if os.path.exists(options.save_main_path): if os.path.exists(options.save_main_path+"temp/") is False: os.makedirs(options.save_main_path+"temp/") sample_save_path = options.save_main_path +"temp/svm.train" train_model.cons_train_sample_for_cla(args[0],indexes,options.local_fun,options.dic_path,sample_save_path,delete=True,str_splitTag=options.str_splitTag,tc_splitTag=options.tc_splitTag) if step==4: search_result_save_path = options.save_main_path +"temp/"+"svm.param" tms_svm.set_svm_type(options.svm_type) if options.svm_type=="libsvm": coarse_c_range=(-5,7,2) coarse_g_range=(3,-10,-2) fine_c_step=0.5 fine_g_step=0.5 if options.svm_type =="liblinear": coarse_c_range=(-5,7,2) coarse_g_range=(1,1,1) fine_c_step=0.5 fine_g_step=0 c,g=grid_search_param.grid(args[0],search_result_save_path,options.svm_type,coarse_c_range,coarse_g_range,fine_c_step,fine_g_step) print "best c = %s\t g = %s\n"%(c,g) if step==5: model_save_path = options.save_main_path+"model/"+options.model_name train_model.ctm_train_model(options.problem_save_path,svm_param,model_save_path)