def hash(features, num_train_samples=58000, L=8):
bits = []
for i in range(L):
start = timeit.default_timer()
m = liblinearutil.load_model(
'models/tr{0:05d}-L{1:02d}-b{2:02d}.model'.format(
num_train_samples, L, i))
p_label, p_acc, p_val = liblinearutil.predict([0] * features.shape[0],
features.tolist(), m,
str('-q'))
bits.append(p_label)
end = timeit.default_timer()
print('[HASH] {0:3d}th bit hashed. {1:.4f} seconds elapsed'.format(
i, end - start))
start = timeit.default_timer()
bits = np.vstack(bits).transpose().astype(np.int)
bits[np.nonzero(bits == 0)] = -1
with open('hash/tr{0:05d}-L{1:02d}'.format(num_train_samples, L),
'wb') as fo:
cPickle.dump(bits, fo)
end = timeit.default_timer()
print('[HASH] Hash codes saved. {0:.4f} seconds elapsed'.format(end -
start))
return
def load(self, model_path):
"""
Loads the model from the directory at the given path
@type model_path: str
@param model_path: path to load the trained model
"""
# load feature_extractor object (used to extract features for the test set)
## if-else statement added on 06.02.2017
# if (self.__feature_extractor.feature_template == "relational") and (self.__feature_extractor.parser_type == "spacy"):
# print("Relational model with spaCy parser cannot be loaded")
# else:
# self.__feature_extractor = pickle.load(open(os.path.join(model_path, "feature_extractor"), "rb"))
# # load trained model
# self.__liblinear_model = liblinearutil.load_model(os.path.join(model_path, "liblinear_model"))
try:
print(self.model_path)
self.__feature_extractor = pickle.load(
open(os.path.join(self.model_path, "feature_extractor"), "rb"))
# load trained model
self.__liblinear_model = liblinearutil.load_model(
os.path.join(self.model_path, "liblinear_model"))
except:
raise Exception("No files found in the model path " +
self.model_path)
def load(path): obj = SVM() f = open(path, 'rb') obj._labels, obj._features = pickle.load(f) f.close() obj._model = liblinear.load_model(path + '-model') return obj
def predictTraitValue(self,
imagelist_file='image_list.txt',
class_label=1,
outfile='output',
norm=-1,
debug=False):
"""
Calls external functions to obtain predicted value between 0 and 1 for given image, updates class values
Args:
imagelist_file: file that contains the list of image/images to be processed
class_label: label given images as either positive or negative for SVM classification
outfile: filename of output for Dense SIFT analysis
Return: True if successful
"""
with open(imagelist_file, 'w') as f:
f.write('tmp.jpg\n')
cv2.imwrite('tmp.jpg', self.norm_image)
hog_histogram(imagelist_file, class_label, outfile, norm, debug)
prob_y, prob_x = svm_read_problem(outfile)
model_file = models[self.trait]
model = load_model(model_file)
self.p_val = predict(prob_y, prob_x, model)[2][0][0]
return True
def __init__(self,
model_filename,
features_filename,
output_probability=False):
self.model = liblinearutil.load_model(model_filename)
self.feature_indices = self._load_features(features_filename)
self.output_probability = output_probability
def load(path):
obj = SVM()
f = open(path, 'rb')
obj._labels, obj._features = pickle.load(f)
f.close()
obj._model = liblinear.load_model(path + '-model')
return obj
def load_model(self , path) :
logging.info("load model of %s" % self.name)
self.model = liblinearutil.load_model(path)
model_ext = self._load_svm_model(".".join([path , "ext"]))
self.feature_dict = model_ext["feature_dict"]
self.gram_n = model_ext["gram_n"]
logging.info("done")
def load_svmmodel(self, filepath):
"""
load svm model
"""
if(os.path.exists(filepath)):
self._model = load_model(filepath)
else:
raise IOError(filepath + ': Model file not found.')
def load_model(self , path) :
logging.info("load model of %s" % self.name)
self.model = liblinearutil.load_model(path)
model_ext = self._load_svm_model(".".join([path , "ext"]))
self.words2idx = model_ext["words2idx"]
self.words_idf = model_ext["words_idf"]
self.gram_n = model_ext["gram_n"]
logging.info("done.")
def load(name):
'''Loads the model and data.
Columns is a dictionnary used when the feature keys were converted to
numbers, for liblinear.
'''
model = llb.load_model('models/{}.model'.format(name))
columns = dts.Dataset.from_columns(name)._features
return model, columns
def batchProcess(imgPath, outPath, opts):
# read eDN model(s)
descFile = open(opts.descPath, 'r')
desc = pickle.load(descFile)
descFile.close()
nFeatures = np.sum([
d['desc'][-1][0][1]['initialize']['n_filters'] for d in desc
if d != None
])
# load SVM model and whitening parameters
svm = load_model(opts.svmPath)
f = open(opts.whitePath, 'r')
whiteParams = np.asarray([map(float, line.split(' ')) for line in f]).T
f.close()
# assemble svm model
svmModel = {}
svmModel['svm'] = svm
svmModel['whitenParams'] = whiteParams
biasToCntr = (svm.get_nr_feature() - nFeatures) == 1
model = EDNSaliencyModel(desc, svmModel, biasToCntr)
puts(colored.yellow("Collecting image filenames..."))
todo_list = [f for f in listdir(imgPath) if isfile(join(imgPath, f))]
existing_list = [f for f in listdir(outPath) if isfile(join(outPath, f))]
#image_list.sort()
puts("Done")
for f in todo_list:
if f not in existing_list:
print '-------------------------------------------------------------'
print 'processing', f
print
try:
# read image
img = misc.imread(join(imgPath, f))
# compute saliency map
salMap = model.saliency(img, normalize=False)
# normalize and save the saliency map to disk
normSalMap = (255.0 / (salMap.max() - salMap.min()) *
(salMap - salMap.min())).astype(np.uint8)
misc.imsave(join(outPath, f), normSalMap)
except Exception as e:
puts(colored.red('ERROR: {}'.format(e)))
puts(colored.yellow('filename: {}'.format(f)))
else:
print '-------------------------------------------------------------'
puts(colored.yellow('Saliency map already exists! ({})'.format(f)))
print
def liblinear_classifier(svm_input=None, y=[], x=[]):
"""调用训练好的liblinear分类器做垃圾过滤
"""
svm_model = load_model(SVM_MODEL_FILE)
if svm_input:
y, x = svm_read_problem(svm_input)
p_label, p_acc, p_val = predict(y, x, svm_model, "-q")
return p_label
def liblinear_classifier(svm_input=None, y=[], x=[]):
"""调用训练好的liblinear分类器做垃圾过滤
"""
svm_model = load_model(SVM_MODEL_FILE)
if svm_input:
y, x = svm_read_problem(svm_input)
p_label, p_acc, p_val = predict(y, x, svm_model, "-q")
return p_label
def __init__(self, featureSet, options):
self.featureSet = featureSet
self.params = '-b 1'
self.lmw = options['lmw']
modelName = options['modelName']
sys.stderr.write('loading transition model...')
self.transProbs = Bigram.getModelFromFile(options['bigramModelFile'])
sys.stderr.write('done\nloading observation model...')
self.model = load_model('{0}.model'.format(modelName))
self.labelCounter = options['labelCounter']
self.featCounter = options['featCounter']
sys.stderr.write('done\n')
def batchProcess(imgPath, outPath, opts):
# read eDN model(s)
descFile = open(opts.descPath, 'r')
desc = pickle.load(descFile)
descFile.close()
nFeatures = np.sum([d['desc'][-1][0][1]['initialize']['n_filters']
for d in desc if d != None])
# load SVM model and whitening parameters
svm = load_model(opts.svmPath)
f = open(opts.whitePath, 'r')
whiteParams = np.asarray([map(float, line.split(' ')) for line in f]).T
f.close()
# assemble svm model
svmModel = {}
svmModel['svm'] = svm
svmModel['whitenParams'] = whiteParams
biasToCntr = (svm.get_nr_feature()-nFeatures) == 1
model = EDNSaliencyModel(desc, svmModel, biasToCntr)
puts(colored.yellow("Collecting image filenames..."))
todo_list = [ f for f in listdir(imgPath) if isfile(join(imgPath,f)) ]
existing_list = [ f for f in listdir(outPath) if isfile(join(outPath,f)) ]
#image_list.sort()
puts("Done")
for f in todo_list:
if f not in existing_list:
print '-------------------------------------------------------------'
print 'processing', f
print
try:
# read image
img = misc.imread(join(imgPath,f))
# compute saliency map
salMap = model.saliency(img, normalize=False)
# normalize and save the saliency map to disk
normSalMap = (255.0 / (salMap.max()-salMap.min()) *
(salMap-salMap.min())).astype(np.uint8)
misc.imsave(join(outPath,f), normSalMap)
except Exception as e:
puts(colored.red('ERROR: {}'.format(e)))
puts(colored.yellow('filename: {}'.format(f)))
else:
print '-------------------------------------------------------------'
puts(colored.yellow('Saliency map already exists! ({})'.format(f)))
print
def __init__(self, featureSet, options):
self.featureSet = featureSet
self.params = '-b 1'
self.lmw = options['lmw']
modelName = options['modelName']
sys.stderr.write('loading transition model...')
self.transProbs = Bigram.getModelFromFile(options['bigramModelFile'])
sys.stderr.write('done\nloading observation model...')
self.model = load_model('{0}.model'.format(modelName))
self.labelCounter = options['labelCounter']
self.featCounter = options['featCounter']
sys.stderr.write('done\n')
def test( C, Y_test, X_test, x_lines ):
"""
This function takes in the test labels and features and prints out the accuracy
:param C : list containing parameter C
:param X_test : test features
:param Y_test : test labels
:return None
"""
# for c in C:
model = lu.load_model("model/lmods2_tamper" + str(round(C,2)) + "_" + str(x_lines) + "l.model")
p_letters, p_acc, p_val = lu.predict(Y_test, X_test, model)
return p_letters
def eDNsaliency(imgPath, outPath, opts):
# read image
img = misc.imread(imgPath)
# read eDN model(s)
descFile = open(opts.descPath, 'r')
desc = pickle.load(descFile)
descFile.close()
nFeatures = np.sum([
d['desc'][-1][0][1]['initialize']['n_filters'] for d in desc
if d != None
])
# load SVM model and whitening parameters
svm = load_model(opts.svmPath)
f = open(opts.whitePath, 'r')
whiteParams = np.asarray([map(float, line.split(' ')) for line in f]).T
f.close()
# assemble svm model
svmModel = {}
svmModel['svm'] = svm
svmModel['whitenParams'] = whiteParams
biasToCntr = (svm.get_nr_feature() - nFeatures) == 1
# compute saliency map
model = EDNSaliencyModel(desc, svmModel, biasToCntr)
salMap = model.saliency(img, normalize=False)
salMap = salMap.astype('f')
if not opts.noBlur:
salMap = ndimage.gaussian_filter(salMap, sigma=30)
# read fixation map / empirical saliency map
if opts.fixMap:
fixMap = misc.imread(opts.fixMap)
# compute AUC
if opts.auc:
auc = evaluate_sal_map(salMap, fixMap)
logging.info("AUC = %f" % auc)
# for fair visual comparison, perform histogram equalization with
# empirical saliency map
if opts.histeq:
salMap = hist_equalize_maps(fixMap, salMap)
# normalize and save the saliency map to disk
normSalMap = (255.0 / (salMap.max() - salMap.min()) *
(salMap - salMap.min())).astype(np.uint8)
misc.imsave(outPath, normSalMap)
def load_model(lang):
"""
Loads the model from file and returns it.
Models are stored in the resources directory
"""
print "Using", lang, "linear model.."
if lang:
return linu.load_model(os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'resources',
lang + '_sentiment.model'))
else:
raise Exception
def eDNsaliency(imgPath, outPath, opts):
# read image
img = misc.imread(imgPath)
# read eDN model(s)
descFile = open(opts.descPath, 'r')
desc = pickle.load(descFile)
descFile.close()
nFeatures = np.sum([d['desc'][-1][0][1]['initialize']['n_filters']
for d in desc if d != None])
# load SVM model and whitening parameters
svm = load_model(opts.svmPath)
f = open(opts.whitePath, 'r')
whiteParams = np.asarray([map(float, line.split(' ')) for line in f]).T
f.close()
# assemble svm model
svmModel = {}
svmModel['svm'] = svm
svmModel['whitenParams'] = whiteParams
biasToCntr = (svm.get_nr_feature()-nFeatures) == 1
# compute saliency map
model = EDNSaliencyModel(desc, svmModel, biasToCntr)
salMap = model.saliency(img, normalize=False)
salMap = salMap.astype('f')
if not opts.noBlur:
salMap = ndimage.gaussian_filter(salMap, sigma=30)
# read fixation map / empirical saliency map
if opts.fixMap:
fixMap = misc.imread(opts.fixMap)
# compute AUC
if opts.auc:
auc = evaluate_sal_map(salMap, fixMap)
logging.info("AUC = %f" % auc)
# for fair visual comparison, perform histogram equalization with
# empirical saliency map
if opts.histeq:
salMap = hist_equalize_maps(fixMap, salMap)
# normalize and save the saliency map to disk
normSalMap = (255.0 / (salMap.max()-salMap.min()) *
(salMap-salMap.min())).astype(np.uint8)
misc.imsave(outPath, normSalMap)
def predict_liblinear(args):
model_name, files = args[0], args[1:]
model = load_model(model_name)
file_objs = [open(fn) for fn in files]
while True:
lines = [f.readline() for f in file_objs]
if '' in lines:
assert len(set(lines)) == 1
break
vector = [float(line.strip()) for line in lines]
xi, max_idx = gen_feature_nodearray(vector)
label = liblinear.predict(model, xi)
#p_label, p_acc, p_val = predict([1], [vector], model)
print class_to_num(label)
def predict_liblinear(args):
model_name, files = args[0], args[1:]
model = load_model(model_name)
file_objs = [open(fn) for fn in files]
while True:
lines = [f.readline() for f in file_objs]
if '' in lines:
assert len(set(lines)) == 1
break
vector = [float(line.strip()) for line in lines]
xi, max_idx = gen_feature_nodearray(vector)
label = liblinear.predict(model, xi)
#p_label, p_acc, p_val = predict([1], [vector], model)
print class_to_num(label)
def LoadModel(self):
if(self.classifierType == "SVM" and self.packageType == "liblinear"):
from liblinearutil import load_model
self.classifierModel = load_model(self.featuresSerializationFileName)
elif(self.classifierType == "DecisionTree" and self.packageType == "nltk"):
# Load the model
serializationFile = open(self.featuresSerializationFileName, 'rb')
self.classifierModel = pickle.load(serializationFile)
serializationFile.close()
else:
print("Only SVM with liblinear is supported to LoadModel")
'''
def LoadModel(self, model_dir):
# load config
input = codecs.open(os.path.join(model_dir, "config.json"), "r", "utf-8")
config_json = json.load(input)
input.close()
self.attrs = config_json["attrs"]
# load feature
self.feature = attr_feature()
self.feature.load_Lexicon(os.path.join(model_dir, config_json["feature_lexicon_file"]))
if not self.feature.is_set:
raise Exception("Fail to load feature module!")
# load svm model
for attr in self.attrs:
self.models[attr] = load_model(os.path.join(model_dir, "%s.svm.m" % (attr)))
self.is_set = True
def LoadModel(self):
if (self.classifierType == "SVM" and self.packageType == "liblinear"):
from liblinearutil import load_model
self.classifierModel = load_model(
self.featuresSerializationFileName)
elif (self.classifierType == "DecisionTree"
and self.packageType == "nltk"):
# Load the model
serializationFile = open(self.featuresSerializationFileName, 'rb')
self.classifierModel = pickle.load(serializationFile)
serializationFile.close()
else:
print("Only SVM with liblinear is supported to LoadModel")
'''
def LoadModel(self, model_dir):
# load config
input = codecs.open(os.path.join(model_dir,'config.json'), 'r', 'utf-8')
config_json = json.load(input)
input.close()
self.slots = config_json['slots']
# load ontology
self.ontology_file = os.path.join(model_dir,config_json['ontology_file'])
self.tagsets = ontology_reader.OntologyReader(self.ontology_file).get_tagsets()
# load feature
self.feature = feature(self.tagsets)
self.feature.load_Lexicon(os.path.join(model_dir,config_json['feature_lexicon_file']))
if not self.feature.is_set:
raise Exception('Fail to load feature module!')
# load svm model
for slot in self.slots:
self.models[slot] = load_model(os.path.join(model_dir, '%s.svm.m' %(slot)))
self.is_set = True
def predictTraitValue(self, imagelist_file='image_list.txt', class_label=1, outfile='output', norm=-1, debug=False):
"""
Calls external functions to obtain predicted value between 0 and 1 for given image, updates class values
Args:
imagelist_file: file that contains the list of image/images to be processed
class_label: label given images as either positive or negative for SVM classification
outfile: filename of output for Dense SIFT analysis
Return: True if successful
"""
with open(imagelist_file, 'w') as f:
f.write('tmp.jpg\n')
cv2.imwrite('tmp.jpg', self.norm_image)
hog_histogram(imagelist_file, class_label, outfile, norm, debug)
prob_y, prob_x = svm_read_problem(outfile)
model_file = models[self.trait]
model = load_model(model_file)
self.p_val = predict(prob_y, prob_x, model)[2][0][0]
return True
def liblinear_predict(problem_filepath, model_filepath):
"""
Using LibLinear to predict result of a problem
Returns
-------
(ids, labels)
"""
# Reading a problem
ids, x = liblinearutil.svm_read_problem(problem_filepath)
print "len(x) = ", len(x)
# Preparing a model
model = liblinearutil.load_model(model_filepath)
# Predicting
y = [-2] * len(x)
p_label, p_acc, p_val = liblinearutil.predict(y, x, model)
return (ids, p_label)
def load(self, modelname, featuresname):
self._svm_model = svm.load_model(modelname)
self._features.load(open(featuresname, 'rb'))
def load(self, model_path):
"""Loads the model from the directory at the given path."""
self.__feature_extractor = pickle.load(
open(os.path.join(model_path, "feature_extractor"), "rb"))
self.__liblinear_model = liblinearutil.load_model(
os.path.join(model_path, "liblinear_model"))
def readScales(scalefile):
scales = {}
with open(scalefile) as f:
for line in f:
k, v = line.strip().split("\t")
scales[int(k)] = float(v)
f.close()
return scales
brnclst = utils.readMetaOptimizeBrownCluster()
embeddings = utils.readMetaOptimizeEmbeddings()
brnspace = initBrnSpace()
scales_shallow = readScales(SHALLOWSCALEFILE)
scales_neuralbrn = readScales(NEURALBRNSCALEFILE)
model_shallow = ll.load_model(SHALLOWMODELFILE)
model_neuralbrn = ll.load_model(NEURALBRNMODELFILE)
def simpleScale(x, trainmaxes=None):
maxes = trainmaxes if trainmaxes != None else {}
if trainmaxes == None:
for itemd in x:
for k, v in itemd.items():
if k not in maxes or maxes[k] < abs(v): maxes[k] = abs(v)
newx = []
for itemd in x:
newd = dict.fromkeys(itemd)
for k, v in itemd.items():
if k in maxes and maxes[k] != 0: newd[k] = (v + 0.0) / maxes[k]
else: newd[k] = 0.0
def predict(instance_file, model_file, param):
y, x = ll.svm_read_problem(instance_file)
prob = ll.problem(y, x)
m = ll.load_model(model_file)
dist = ll.predict(y, x, m, param)[2]
print dist[0]
def load(self, modelname, featuresname): self._svm_model = svm.load_model(modelname) self._features.load(open(featuresname, 'rb'))
def load(self, model_path):
"""Loads the model from the directory at the given path."""
self.__feature_extractor = pickle.load(
open(os.path.join(model_path, "feature_extractor"), "rb"))
self.__liblinear_model = liblinearutil.load_model(
os.path.join(model_path, "liblinear_model"))
def predict(input_file, model0_file, model1_file, mapping_file, output_file):
global bos, eos
out = open(output_file, 'w')
m0 = ll.load_model(model0_file)
m1 = ll.load_model(model1_file)
mapping = Mapping(mapping_file)
bos = mapping.map_pos('BOS')
eos = mapping.map_pos('EOS')
print '# of features:', len(mapping.feature_dict)
# for easier mapping from neighbouring pos tags to features
p_f = {}
for i in mapping.pos_dict_rev: #[0, 1, 2, 3, ...] bos and eos are also included
pi = mapping.map_pos_rev(i)
p_f[(-1, i)] = mapping.map_features('POS_P1:%s' % pi)
p_f[(-2, i)] = mapping.map_features('POS_P2:%s' % pi)
p_f[(+1, i)] = mapping.map_features('POS_N1:%s' % pi)
p_f[(+2, i)] = mapping.map_features('POS_N2:%s' % pi)
for j in mapping.pos_dict_rev:
pj = mapping.map_pos_rev(j)
p_f[(-1, -2, i, j)] = mapping.map_features('POS_P1_P2:%s_%s' % (pi, pj))
p_f[(+1, +2, i, j)] = mapping.map_features('POS_N1_N2:%s_%s' % (pi, pj))
p_f[(-1, +1, i, j)] = mapping.map_features('POS_P1_N1:%s_%s' % (pi, pj))
s0, s1, s2 = 0, 0, 0
total = 0
for sent in read_sentence(input_file):
x_ = []
g_ = []
for t in sent:
feat = t.maxent_features(mapping.map_features)
x_.append(feat)
g_.append(mapping.map_pos(t.gold_pos))
y_0 = map(int, ll.predict([], [{k : 1 for k in f} for f in x_], m0, '-q')[0])
# y_2 = [choice(xrange(1, len(mapping.pos_dict))) for i in y_1]
y_1, y_2 = inference(m1, p_f, y_0[:], x_, propose_deterministic)
# y_2 = inference(m1, p_f, y_1[:], x_, propose_probabilistic)
for y, y0, y1, y2, t in zip(g_, y_0, y_1, y_2, sent):
if y == y0:
s0 += 1
if y == y1:
s1 += 1
if y == y2:
s2 += 1
total += 1
p0 = mapping.map_pos_rev(y0)
p1 = mapping.map_pos_rev(y1)
p2 = mapping.map_pos_rev(y2)
out.write('%s\t%s\n' % (t.word, p1))
out.write('\n')
out.close()
print 'acc 0: %d / %d = %.4f' % (s0, total, s0 / total)
print 'acc 1: %d / %d = %.4f' % (s1, total, s1 / total)
print 'acc 2: %d / %d = %.4f' % (s2, total, s2 / total)
def __init__(self):
## load SIFT feature extractor
if cv2.__version__[0] == '2':
self.sift_extractor = cv2.DescriptorExtractor_create("SIFT")
else:
self.sift_extractor = cv2.xfeatures2d.SIFT_create()
module_path = os.path.dirname(os.path.realpath(__file__))
self.model_path = os.path.join(module_path, 'model')
## construct face detector
self.face_detector = {}
self.face_detector['detector'] = cv2.CascadeClassifier(os.path.join(self.model_path, 'haarcascade_frontalface_alt2.xml'))
self.face_detector['minNeighbors'] = 4
self.face_detector['minSize'] = (20, 20)
# confidence LBP
transform_svm_mat2file(os.path.join(self.model_path, 'confidence_face_LBP.mat'))
self.face_detector['confidence_LBP'] = {'model':load_model(os.path.join(self.model_path, 'confidence_face_LBP')),
'thre': 0.01}
# confidence SIFT
matdata = loadmat(os.path.join(self.model_path, 'meanImg_sing_20.mat'))
kpt_small = []
kpt_small.append(cv2.KeyPoint(20, 20, 3.1, -1, 1, 0, 1))
descriptors_mean_small = np.zeros((1, 0))
for idx_pt in np.arange(17, 68):
_, temp = self.sift_extractor.compute(matdata['meanImg'][idx_pt]['roi'], kpt_small)
descriptors_mean_small = np.concatenate((descriptors_mean_small, temp.reshape((1, -1))), axis=1)
# Lower the confidence, higher the tolerance.
# For this version, we can set different threhsolds for detection and tracking phases.
# For near-frontal face, the confidence is about 0.65.
# For non-frontal faces the confidence is about 0.55.
self.face_detector['confidence_SIFT'] = {'descriptor':descriptors_mean_small,
'thre_detect': 0.56,
'thre_track': 0.46}
## construct landmark detector/tracker
self.face_lmks_model = {}
prior = loadmat(os.path.join(self.model_path, 'Prior.mat'))
mm = prior['mm'].reshape(-1, 2)
self.face_lmks_model['mm'] = mm[range(17, 68), :].flatten()
self.face_lmks_model['num_pts'] = 51
self.face_lmks_model['num_iter'] = 4
self.face_lmks_model['norm_width'] = 200
self.face_lmks_model['margin'] = 50
self.face_lmks_model['para_detect'] = {}
self.face_lmks_model['para_track'] = {}
for it in range(self.face_lmks_model['num_iter']):
x = loadmat(os.path.join(self.model_path, 'Detect_it%d.mat'%(it+1)))
self.face_lmks_model['para_detect'][it] = x
x = loadmat(os.path.join(self.model_path, 'Tracking_it%d.mat'%(it+1)))
self.face_lmks_model['para_track'][it] = x
## face motion parameters
self.face_motion = {'queue_size': 60,
'threshold': 10,
'queue':100*np.random.normal(size=(60, self.face_lmks_model['num_pts']*2))}
## construct eye detector
self.eye_detector_SDM = {}
self.eye_detector_SDM['norm_width'] = 25
self.eye_detector_SDM['num_pts'] = 27
self.eye_detector_SDM['num_iter'] = 3
matdata = loadmat(os.path.join(self.model_path, 'leftPara_20160516.mat'))
mmc = matdata['leftmm'].reshape((-1, 2))
pair_set = np.array([[i, j] for i in np.arange(self.eye_detector_SDM['num_pts']) for j in np.arange(i+1, self.eye_detector_SDM['num_pts'])])
pt1 = mmc[pair_set[:, 0], :]
pt2 = mmc[pair_set[:, 1], :]
mmshapefea1 = (pt1 - pt2).reshape((1, -1))
mmshapefea2 = np.sqrt(np.sum((pt1 - pt2)**2, axis=1))
self.eye_detector_SDM['pair_set'] = pair_set
self.eye_detector_SDM['mmshapefea1'] = mmshapefea1
self.eye_detector_SDM['mmshapefea2'] = mmshapefea2
#for it in range(self.eye_detector_SDM['num_iter']):
self.eye_detector_SDM['para'] = matdata['leftPara']
self.eye_detector_SDM['mm'] = matdata['leftmm']
def readScales(scalefile):
scales = {}
with open(scalefile) as f:
for line in f:
k,v = line.strip().split("\t")
scales[int(k)] = float(v)
f.close()
return scales
brnclst = utils.readMetaOptimizeBrownCluster()
embeddings = utils.readMetaOptimizeEmbeddings()
brnspace = initBrnSpace()
scales_shallow = readScales(SHALLOWSCALEFILE)
scales_neuralbrn = readScales(NEURALBRNSCALEFILE)
model_shallow = ll.load_model(SHALLOWMODELFILE)
model_neuralbrn = ll.load_model(NEURALBRNMODELFILE)
def simpleScale(x, trainmaxes=None):
maxes = trainmaxes if trainmaxes!=None else {}
if trainmaxes == None:
for itemd in x:
for k,v in itemd.items():
if k not in maxes or maxes[k] < abs(v): maxes[k] = abs(v)
newx = []
for itemd in x:
newd = dict.fromkeys(itemd)
for k,v in itemd.items():
if k in maxes and maxes[k] != 0: newd[k] = (v+0.0)/maxes[k]
else: newd[k] = 0.0
newx.append(newd)
import os
pkg_path = os.environ["geoloc"]
import ujson as json
import xmlrpclib
from geoloc.adapters import twitter_adapter, feature_adapter, city_adapter
from geoloc.util import lib_grid_search, gcd_dist, lib_log
import liblinearutil
fea_num = city_adapter.get_feature_number()
#NOTE: Only text, loc, tz models are adopted, as they achieve comparable performance, but consumes much less memory than using the full model.
text_decoder = xmlrpclib.ServerProxy("http://localhost:9001")
loc_decoder = xmlrpclib.ServerProxy("http://localhost:9002")
tz_decoder = xmlrpclib.ServerProxy("http://localhost:9003")
#desc_decoder = xmlrpclib.ServerProxy("http://localhost:9004")
#rname_decoder = xmlrpclib.ServerProxy("http://localhost:9005")
stacked_model = liblinearutil.load_model(
'{0}/models/world.l1.train.model.text_loc_tz'.format(pkg_path))
#stacked_model = liblinearutil.load_model('{0}/models/world.l1.train.model.text_loc_tz_desc_rname'.format(pkg_path))
print "Geolocation models loaded"
def seek_cache(sname):
""" Seek on disk cache (predicted GT-JSON) """
jobj = None
try:
assert (sname)
cache_file = "{0}/cache/{1}".format(pkg_path, sname)
jstr = open(cache_file).readline()
jobj = json.loads(jstr.rstrip())
except (AssertionError, IOError, ValueError):
return None
return jobj
pkg_path = os.environ["geoloc"]
import ujson as json
import xmlrpclib
from geoloc.adapters import twitter_adapter, feature_adapter, city_adapter
from geoloc.util import lib_grid_search, gcd_dist, lib_log
import liblinearutil
fea_num = city_adapter.get_feature_number()
#NOTE: Only text, loc, tz models are adopted, as they achieve comparable performance, but consumes much less memory than using the full model.
text_decoder = xmlrpclib.ServerProxy("http://localhost:9001")
loc_decoder = xmlrpclib.ServerProxy("http://localhost:9002")
tz_decoder = xmlrpclib.ServerProxy("http://localhost:9003")
#desc_decoder = xmlrpclib.ServerProxy("http://localhost:9004")
#rname_decoder = xmlrpclib.ServerProxy("http://localhost:9005")
stacked_model = liblinearutil.load_model('{0}/models/world.l1.train.model.text_loc_tz'.format(pkg_path))
#stacked_model = liblinearutil.load_model('{0}/models/world.l1.train.model.text_loc_tz_desc_rname'.format(pkg_path))
print "Geolocation models loaded"
def seek_cache(sname):
""" Seek on disk cache (predicted GT-JSON) """
jobj = None
try:
assert(sname)
cache_file = "{0}/cache/{1}".format(pkg_path, sname)
jstr = open(cache_file).readline()
jobj = json.loads(jstr.rstrip())
except (AssertionError, IOError, ValueError):
return None
return jobj
def train(self):
if os.path.isfile("svm.model") and self.useModel:
self.model = llu.load_model("svm.model")
else:
self.model = llu.train(self.ys, self.xs, self.train_param)
llu.save_model("svm.model", self.model)
def load(path):
obj = ml.Classifier.load(path)
obj._model = liblinear.load_model(path + "-model")
return obj
def compute_eDN_saliency_map(
img,
no_normalization=True,
eDN_desc_path='edn_cvpr2014/slmBestDescrCombi.pkl',
eDN_svm_path='edn_cvpr2014/svm-slm',
eDN_whitening_params_path='edn_cvpr2014/whiten-slm',
verbose=False,
**kwargs):
# determine the image dimensions
proportions = float(img.shape[1]) / img.shape[0]
if abs(proportions - 2.0) < 1e-5:
# 2:1
insize = (768, 384)
elif abs(proportions - 1.0) < 1e-5:
# 1:1
insize = (512, 512)
elif abs(proportions - 6.0 / 5.0) < 1e-5:
# 6:5
insize = (600, 500)
elif abs(proportions - 4.0 / 3.0) < 1e-5:
# 4:3
insize = (600, 450)
else:
# default, 4:3, smaller
insize = (512, 384)
if verbose:
print >> sys.stderr, 'Choosing internal image resolution of', insize
# read eDN model(s)
descFile = open(eDN_desc_path, 'r')
desc = pickle.load(descFile)
descFile.close()
nFeatures = np.sum([
d['desc'][-1][0][1]['initialize']['n_filters'] for d in desc
if d != None
])
# load SVM model and whitening parameters
svm = load_model(eDN_svm_path)
f = open(eDN_whitening_params_path, 'r')
white_params = np.asarray([map(float, line.split(' ')) for line in f]).T
f.close()
# assemble svm model
svm_model = {}
svm_model['svm'] = svm
svm_model['whitenParams'] = white_params
bias_to_center_validation = (svm.get_nr_feature() - nFeatures) == 1
assert bias_to_center_validation == False
# compute saliency map
model = EDNSaliencyModel(descriptions=desc,
svmModel=svm_model,
biasToCntr=False,
insize=insize)
sal_map = model.saliency(img, normalize=False)
sal_map = sal_map.astype('f')
if not no_normalization:
sal_map = (255.0 / (sal_map.max() - sal_map.min()) *
(sal_map - sal_map.min())).astype(np.uint8)
return sal_map
def load(self, model_file):
model = liblinearutil.load_model(model_file)
os.environ['GLOG_minloglevel'] = '3' # Suppress logging.
if __name__ == "__main__":
desc_file_path = 'slmBestDescrCombi.pkl'
with open(desc_file_path) as fp:
desc = pickle.load(fp)
nFeatures = np.sum([
d['desc'][-1][0][1]['initialize']['n_filters'] for d in desc
if d != None
])
# load SVM model and whitening parameters
svm_path = 'svm-slm-cntr'
svm = load_model(svm_path)
whiten_path = 'whiten-slm-cntr'
with open(whiten_path) as fp:
whitenParams = np.asarray([map(float, line.split(' '))
for line in fp]).T
# assemble svm model
svmModel = {'svm': svm, 'whitenParams': whitenParams}
biasToCntr = (svm.get_nr_feature() - nFeatures) == 1
def eDNsaliency(image_path):
img = misc.imread(image_path, mode='RGB')
# compute saliency map
def load(path):
obj = ml.Classifier.load(path)
obj._model = liblinear.load_model(path + '-model')
return obj
def multipule_eval_for_logistic(test_corpus_dir, feature_map_character, feature_map_numeric, feature_show, args):
"""
liblinearのlogisticで作成したモデル（一文ごとにラベルを判断）を行う
一回でも+1が発生すれば，文書にラベルが付与されたと見なす
"""
env = "pine"
if env == "pine":
# change below by an environment
libsvm_wrapper_path = "/home/kensuke-mi/opt/libsvm-3.17/python/"
elif env == "local":
libsvm_wrapper_path = "/Users/kensuke-mi/opt/libsvm-3.17/python/"
liblinear_wrapper_path = "/Users/kensuke-mi/opt/liblinear-1.94/python/"
sys.path.append(liblinear_wrapper_path)
sys.path.append(libsvm_wrapper_path)
import liblinearutil
import svmutil
if args.save_performance == True:
performance_out = codecs.open("./performance_result." + args.experiment_no, "w", "utf-8")
performance_out.write(args.experiment_no + u"\n")
performance_out.write(u"-" * 30 + u"\n")
# 確信度の閾値
threshold = float(args.threshold)
# 確信度を表示するか？オプション
show_confidence = False
# 確信度の平均値
average_confidence = 0
# +1のインスタンス数
times_plus_1_ins = 0
num_docs_having_motif = {}
stop = args.stop
tfidf_flag = args.tfidf
exno = args.experiment_no
model_dir_path = "../get_thompson_motif/classifier/logistic_2nd/"
model_path_list = load_files(model_dir_path, "logistic." + exno)
# 分類器が正しい判断をした回数を保存する．つまりCAP(gold motif tag, candidate by classifier)
num_of_correct_decision = {}
precision_sum = 0
recall_sum = 0
F_sum = 0
h_loss_sum = 0
subset_acc_sum = 0
ex_p_sum = 0
ex_r_sum = 0
ex_f_sum = 0
acc_sum = 0
classifier_return_1_sum = 0
for test_file in load_files(test_corpus_dir):
# ============================================================
result_map = {}
gold_map = {}
# ------------------------------------------------------------
if args.persian_test == True:
# 文ごとにインスタンスの作成
sentences_in_document, motif_stack = file_loader_sentence(test_file, stop)
elif args.dutch_test == True:
sentences_in_document, motif_stack = file_loader_dutch_sentence(test_file, stop)
# ------------------------------------------------------------
out_libsvm_format_sentence(
sentences_in_document, feature_map_character, feature_map_numeric, feature_show, tfidf_flag
)
test_y, test_x = svmutil.svm_read_problem("test.data")
# ------------------------------------------------------------
for model_file in model_path_list:
decision_flag = False
alphabet_label = unicode(os.path.basename(model_file)[0], "utf-8")
result_map[alphabet_label] = 0
model = liblinearutil.load_model(model_file)
p_label, p_acc, p_val = liblinearutil.predict(test_y, test_x, model, "-b 1")
for index, result_label in enumerate(p_label):
if result_label == 1.0:
decision_flag = True
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if decision_flag == True and p_val[index][0] > threshold:
result_map[alphabet_label] = 1
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# ------------------------------------------------------------
for gold_motif in motif_stack:
alphabet_label = gold_motif[0]
gold_map[alphabet_label] = 1
# ------------------------------------------------------------
gold_cap_result = {}
for label in result_map:
if result_map[label] == 1 and label in gold_map:
gold_cap_result[label] = 1
# ------------------------------------------------------------
try:
average = average_confidence / times_plus_1_ins
except ZeroDivisionError:
average = 0
print "-" * 30
print "Filename:{}\nEstimated:{}\nGold:{}\nCorrect Estimation:{}".format(
test_file, result_map, gold_map, gold_cap_result
)
print "average confidence is {}".format(average)
print "-" * 30
# ------------------------------------------------------------
h_loss_sum = calc_h_loss(result_map, gold_map, h_loss_sum)
subset_acc_sum = calc_subset_acc(result_map, gold_map, subset_acc_sum)
ex_p_sum, ex_r_sum, ex_f_sum, acc_sum = calc_p_r_f(result_map, gold_map, ex_p_sum, ex_r_sum, ex_f_sum, acc_sum)
classifier_return_1_sum += get_the_num_of_1_classifier(result_map)
# ============================================================
num_of_files = len(load_files(test_corpus_dir))
h_loss = h_loss_sum / num_of_files
subset_acc = float(subset_acc_sum) / num_of_files
ex_p = ex_p_sum / num_of_files
ex_r = ex_r_sum / num_of_files
ex_f = ex_f_sum / num_of_files
acc = acc_sum / num_of_files
classifier_return_1 = float(classifier_return_1_sum) / num_of_files
precision_ave = precision_sum / len(load_files(test_corpus_dir))
recall_ave = recall_sum / len(load_files(test_corpus_dir))
F_ave = F_sum / len(load_files(test_corpus_dir))
print "-" * 30
print "RESULT for {} files classification".format(len(load_files(test_corpus_dir)))
hamming_format = u"Hamming Loss:{}".format(h_loss)
subset_format = u"Subset Accuracy(classification accuracy):{}".format(subset_acc)
else_format = u"example-based precision:{} example-based recall:{} example-based F:{} accuracy:{}".format(
ex_p, ex_r, ex_f, acc
)
classifier_format = u"Ave. number of classifier which returns 1:{}".format(classifier_return_1)
print hamming_format
print subset_format
print else_format
print classifier_format
if args.save_performance == True:
performance_out.write(hamming_format + u"\n")
performance_out.write(subset_format + u"\n")
performance_out.write(else_format + u"\n")
performance_out.write(classifier_format + u"\n")
performance_out.close()
def init_model(filename="model"): return svm.load_model(filename)
def __init__(self, model_filename, features_filename, output_probability=False):
self.model = liblinearutil.load_model(model_filename)
self.feature_indices = self._load_features(features_filename)
self.output_probability = output_probability
