def process(options, trainCollection, modelAnnotationName, trainAnnotationName, feature): rootpath = options.rootpath modelName = options.model if 'fastlinear' == modelName: from fastlinear.fastlinear import fastlinear_load_model as load_model from fastlinear.fastlinear import fastlinear_save_model as save_model else: from fiksvm.fiksvm import fiksvm_load_model as load_model from fiksvm.fiksvm import fiksvm_save_model as save_model concepts = readConcepts(trainCollection, trainAnnotationName, rootpath) concepts = [concepts[i] for i in range(len(concepts)) if (i%options.numjobs + 1) == options.job] feat_file = BigFile(os.path.join(rootpath, trainCollection, "FeatureData", feature)) for concept in concepts: modelfile = os.path.join(rootpath, trainCollection, 'Models', modelAnnotationName, feature, modelName, '%s.model' % concept) model = load_model(modelfile) (A0, B0) = model.get_probAB() if abs(A0) > 1e-8 and not options.overwrite: printStatus(INFO, "old parameters exist as A=%g, B=%g, skip" % (A0, B0)) continue names,labels = readAnnotationsFrom(trainCollection, trainAnnotationName, concept, skip_0=True, rootpath=rootpath) name2label = dict(zip(names, labels)) results = classify_large_data(model, names, feat_file, prob_output=False) labels = [name2label[x[0]] for x in results] dec_values = [x[1] for x in results] printStatus(INFO, "%s +%d -%d" % (concept, len([x for x in labels if x==1]), len([x for x in labels if x==-1]))) [A,B] = sigmoid_train(dec_values, labels) model.set_probAB(A, B) save_model(modelfile, model) (A1, B1) = model.get_probAB() printStatus(INFO, "A: %g -> %g, B: %g -> %g" % (A0, A1, B0, B1))
def process(options, model_name, concept_file, weight_dir, result_dir): rootpath = options.rootpath overwrite = options.overwrite if 'fastlinear' == model_name: from fastlinear.fastlinear import fastlinear_load_model as load_model from fastlinear.fastlinear import fastlinear_save_model as save_model else: from fiksvm.fiksvm import fiksvm_load_model as load_model from fiksvm.fiksvm import fiksvm_save_model as save_model concepts = [ x.strip() for x in open(concept_file).readlines() if x.strip() and not x.strip().startswith('#') ] todo = [ x for x in concepts if overwrite or not os.path.exists(os.path.join(result_dir, '%s.model' % x)) ] printStatus(INFO, '%d concepts to do' % len(todo)) for concept in todo: weight_file = os.path.join(weight_dir, '%s.txt' % concept) weight_data = map(str.strip, open(weight_file).readlines()) nr_of_models = len(weight_data) assert (nr_of_models >= 2) weights = [0] * nr_of_models models = [None] * nr_of_models for i, line in enumerate(weight_data): w, model_dir = line.split() weights[i] = float(w) model_dir = model_dir if model_dir.startswith( rootpath) else os.path.join(rootpath, model_dir) assert (model_dir.find(model_name) > 0) model_file_name = os.path.join(model_dir, '%s.model' % concept) models[i] = load_model(model_file_name) new_model = models[0] new_model.add_fastsvm(models[1], weights[0], weights[1]) for i in range(2, len(models)): new_model.add_fastsvm(models[i], 1, weights[i]) new_model_file = os.path.join(result_dir, '%s.model' % concept) makedirsforfile(new_model_file) save_model(new_model_file, new_model)
def process(options, model_name, concept_file, weight_dir, result_dir): rootpath = options.rootpath overwrite = options.overwrite if 'fastlinear' == model_name: from fastlinear.fastlinear import fastlinear_load_model as load_model from fastlinear.fastlinear import fastlinear_save_model as save_model else: from fiksvm.fiksvm import fiksvm_load_model as load_model from fiksvm.fiksvm import fiksvm_save_model as save_model concepts = [x.strip() for x in open(concept_file).readlines() if x.strip() and not x.strip().startswith('#')] todo = [x for x in concepts if overwrite or not os.path.exists(os.path.join(result_dir, '%s.model'%x))] printStatus(INFO, '%d concepts to do' % len(todo)) for concept in todo: weight_file = os.path.join(weight_dir, '%s.txt' % concept) weight_data = map(str.strip, open(weight_file).readlines()) nr_of_models = len(weight_data) assert(nr_of_models >= 2) weights = [0] * nr_of_models models = [None] * nr_of_models for i,line in enumerate(weight_data): w, model_dir = line.split() weights[i] = float(w) model_dir = model_dir if model_dir.startswith(rootpath) else os.path.join(rootpath, model_dir) assert (model_dir.find(model_name)>0) model_file_name = os.path.join(model_dir, '%s.model' % concept) models[i] = load_model(model_file_name) new_model = models[0] new_model.add_fastsvm(models[1], weights[0], weights[1]) for i in range(2, len(models)): new_model.add_fastsvm(models[i], 1, weights[i]) new_model_file = os.path.join(result_dir, '%s.model'%concept) makedirsforfile(new_model_file) save_model(new_model_file, new_model)
feat_dir = os.path.join(rootpath, testCollection, "FeatureData", feature) feat_file = BigFile(feat_dir) _renamed, _vectors = feat_file.read(imset) nr_of_images = len(_renamed) nr_of_concepts = len(concepts) mAP = 0.0 models = [None] * len(concepts) stream = StreamFile(feat_dir) for i,concept in enumerate(concepts): model_file_name = os.path.join(rootpath,trainCollection,'Models',trainAnnotationName,feature, modelName, '%s.model'%concept) model = load_model(model_file_name) #print model.get_probAB() models[i] = model names,labels = readAnnotationsFrom(testCollection, testAnnotationName, concept, rootpath=rootpath) name2label = dict(zip(names,labels)) ranklist1 = [(_id, model.predict(_vec)) for _id,_vec in zip(_renamed, _vectors)] stream.open() ranklist2 = [(_id, model.predict(_vec)) for _id,_vec in stream] stream.close() ranklist3 = [(_id, model.predict_probability(_vec)) for _id,_vec in zip(_renamed, _vectors)] print concept,
concepts = readConcepts(testCollection,testAnnotationName,rootpath=rootpath) feat_dir = os.path.join(rootpath, testCollection, "FeatureData", feature) feat_file = BigFile(feat_dir) _renamed, _vectors = feat_file.read(imset) nr_of_images = len(_renamed) nr_of_concepts = len(concepts) mAP = 0.0 models = [None] * len(concepts) for i,concept in enumerate(concepts): model_file_name = os.path.join(rootpath,trainCollection,'Models',trainAnnotationName,feature, modelName, '%s.model'%concept) model1 = load_model(model_file_name) (pA,pB) = model1.get_probAB() model2 = load_model(model_file_name) model2.add_fastsvm(model1, 0.8, 1) names,labels = readAnnotationsFrom(testCollection, testAnnotationName, concept, rootpath=rootpath) name2label = dict(zip(names,labels)) ranklist1 = [(_id, model1.predict(_vec)) for _id,_vec in zip(_renamed, _vectors)] ranklist2 = [(_id, model2.predict(_vec)) for _id,_vec in zip(_renamed, _vectors)] model_file_name = os.path.join(rootpath,trainCollection,'Models', 'bag' + trainAnnotationName,feature, modelName, '%s.model'%concept) model3 = load_model(model_file_name)
_renamed, _vectors = feat_file.read(imset) nr_of_images = len(_renamed) nr_of_concepts = len(concepts) mAP = 0.0 models = [None] * len(concepts) stream = StreamFile(feat_dir) for i, concept in enumerate(concepts): model_file_name = os.path.join(rootpath, trainCollection, 'Models', trainAnnotationName, feature, modelName, '%s.model' % concept) model = load_model(model_file_name) #print model.get_probAB() models[i] = model names, labels = readAnnotationsFrom(testCollection, testAnnotationName, concept, rootpath=rootpath) name2label = dict(zip(names, labels)) ranklist1 = [(_id, model.predict(_vec)) for _id, _vec in zip(_renamed, _vectors)] stream.open() ranklist2 = [(_id, model.predict(_vec)) for _id, _vec in stream] stream.close()
feat_dir = os.path.join(rootpath, testCollection, "FeatureData", feature) feat_file = BigFile(feat_dir) _renamed, _vectors = feat_file.read(imset) nr_of_images = len(_renamed) nr_of_concepts = len(concepts) mAP = 0.0 models = [None] * len(concepts) for i, concept in enumerate(concepts): model_file_name = os.path.join( rootpath, trainCollection, "Models", trainAnnotationName, feature, modelName, "%s.model" % concept ) model1 = load_model(model_file_name) (pA, pB) = model1.get_probAB() model2 = load_model(model_file_name) model2.add_fastsvm(model1, 0.8, 1) names, labels = readAnnotationsFrom(testCollection, testAnnotationName, concept, rootpath=rootpath) name2label = dict(zip(names, labels)) ranklist1 = [(_id, model1.predict(_vec)) for _id, _vec in zip(_renamed, _vectors)] ranklist2 = [(_id, model2.predict(_vec)) for _id, _vec in zip(_renamed, _vectors)] model_file_name = os.path.join( rootpath, trainCollection, "Models", "bag" + trainAnnotationName, feature, modelName, "%s.model" % concept ) model3 = load_model(model_file_name)