def single_execution(config):
"""
Executes training/prediction for the config as a single individual process
:param config: config object
:return:
"""
os.environ["CUDA_VISIBLE_DEVICES"] = str(config.default_gpu)[-1]
from data_processing import CDRSInferenceData, Dataset
from train import Train
from predict import Predict
if config.cdrs_inference:
dataset = CDRSInferenceData(config)
else:
dataset = Dataset(config)
# training
if config.train:
print("Training")
train = Train(config, dataset)
train.train()
# prediction
else:
print("Prediction")
pred = Predict(config, dataset)
pred.predict()
return
def main(args):
init_logger()
set_seed(args)
tokenizer = load_tokenizer(args)
train_dataset = None
dev_dataset = None
test_dataset = None
if args.do_train or args.do_eval:
test_dataset = load_and_cache_examples(args, tokenizer, mode="test")
if args.do_train:
train_dataset = load_and_cache_examples(args, tokenizer, mode="train")
trainer = Trainer(args, train_dataset, dev_dataset, test_dataset)
predictor = Predict(args)
if args.do_train:
trainer.train()
if args.do_eval:
trainer.load_model()
trainer.evaluate("test", "eval")
if args.do_predict:
predictor.predict()
def pred():
p = Predict('text-cnn.model', 100)
#! Not working -_- preds are way off
p.predict([
'Whats up champ', 'call us to get some free goods',
'when will you are going ?'
])
def predict():
param = jsonify((request.data).decode('utf-8'))
param = json.loads(param.json)
param = param['article']
model = Predict(param)
response = jsonify(model.predict())
return response
def main(args):
init_logger()
tokenizer = load_tokenizer(args)
train_dataset = None if args.do_predict else load_and_cache_examples(
args, tokenizer, mode="train")
dev_dataset = None
test_dataset = None if args.do_predict else load_and_cache_examples(
args, tokenizer, mode="test")
if args.do_train:
trainer = Trainer(args, train_dataset, dev_dataset, test_dataset)
trainer.train()
if args.do_eval:
trainer = Trainer(args, train_dataset, dev_dataset, test_dataset)
trainer.load_model()
trainer.evaluate("test")
if args.do_predict:
predict = Predict(args, tokenizer)
predict.load_model()
sentences = [args.sentence]
result_json = dict()
result_json['result'] = int(predict.predict(sentences))
print(json.dumps(result_json, ensure_ascii=False))
def generate_predicted_files():
# set the proper symbol file and model file
symbol_file_path = '../symbol_farm/symbol_10_160_17L_4scales_v1_deploy.json'
model_file_path = '../saved_model/configuration_10_160_17L_4scales_v1_2019-09-20-13-08-26/train_10_160_17L_4scales_v1_iter_800000.params'
my_predictor = Predict(
mxnet=mxnet,
symbol_file_path=symbol_file_path,
model_file_path=model_file_path,
ctx=mxnet.gpu(0),
receptive_field_list=cfg.param_receptive_field_list,
receptive_field_stride=cfg.param_receptive_field_stride,
bbox_small_list=cfg.param_bbox_small_list,
bbox_large_list=cfg.param_bbox_large_list,
receptive_field_center_start=cfg.param_receptive_field_center_start,
num_output_scales=cfg.param_num_output_scales)
# set the val root, the path should look like XXXX/WIDER_val/images
txt_file_path = '/media/heyonghao/HYH-4T-WD/public_dataset/head_detection/brainwash/brainwash/brainwash_test.idl'
image_root = '/media/heyonghao/HYH-4T-WD/public_dataset/head_detection/brainwash/brainwash'
predicted_file_root = './brainwash_testset_predicted_files_for_evaluation_' + os.path.basename(
model_file_path).split('.')[0]
if not os.path.exists(predicted_file_root):
os.makedirs(predicted_file_root)
fin = open(txt_file_path, 'r')
resize_scale = 1
score_threshold = 0.05
NMS_threshold = 0.6
counter = 0
for line in fin:
line = line.strip(';\n')
im_path = re.findall('["](.*?)["]', line)[0]
im = cv2.imread(os.path.join(image_root, im_path), cv2.IMREAD_COLOR)
bboxes = my_predictor.predict(im,
resize_scale=resize_scale,
score_threshold=score_threshold,
top_k=10000,
NMS_threshold=NMS_threshold)
# for bbox in bboxes:
# cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 255, 0), 1)
# cv2.imshow('im',im)
# cv2.waitKey()
predicted_file_name = im_path.replace('/', '_')
predicted_file_name = predicted_file_name.replace('png', 'txt')
fout = open(os.path.join(predicted_file_root, predicted_file_name),
'w')
for bbox in bboxes:
fout.write('head %.03f %d %d %d %d' %
(bbox[4] if bbox[4] <= 1 else 1, math.floor(bbox[0]),
math.floor(bbox[1]), math.ceil(bbox[2] - bbox[0]),
math.ceil(bbox[3] - bbox[1])) + '\n')
fout.close()
counter += 1
print('[%d] is processed.' % counter)
def download_woff(file_name):
# vfile.meituan.net/colorstone/9c846f7774c2b8280bd204adba5c669a2276.woff
url = "https://vfile.meituan.net/colorstone/" + str(file_name) + ".woff"
# print(url)
file_name = str(file_name) + ".woff"
# 这里判断一下 url 的文件是否存在
# 缓存中存在
result = decode_list.get(file_name)
if result:
# 如果存在的话
return str(result)
# 不存在
url = re.sub(" ", "", str(url))
# 把文件下载下来
response = requests.get(url)
download_path_ = str(download_path) + str(file_name)
file_ = open(download_path_, 'wb')
file_.write(response.content)
response.close()
file_.close()
# 预测在这里
result = Predict.predict(download_path_)
# 把预测结果放到缓存数组中去 提速
decode_list[file_name] = result
return str(result)
def semi_supervised(samples_path, write_path, beam_search):
"""use reference to predict source
Args:
samples_path (str): The path of reference
write_path (str): The path of new samples
"""
###########################################
# TODO: module 3 task 1 #
###########################################
pred = Predict()
print('vocab_size:', len(pred.vocab))
count = 0
semi = []
with open(samples_path, 'r') as f:
for picked in f:
count += 1
source, ref = picked.strip().split('<sep>')
prediction = pred.predict(ref.split(), beam_search=beam_search)
# 拼接ref的预测结果与ref，形成新的样本
semi.append(prediction + ' <sep> ' + ref)
if count % 100 == 0:
print(count)
write_samples(semi, write_path, 'a')
semi = []
def semi_supervised(samples_path, write_path, beam_search):
"""use reference to predict source
Args:
samples_path (str): The path of reference
write_path (str): The path of new samples
"""
pred = Predict()
print('vocab_size: ', len(pred.vocab))
# Randomly pick a sample in test set to predict.
count = 0
semi = []
with open(samples_path, 'r') as f:
for picked in f:
count += 1
source, ref = picked.strip().split('<sep>')
prediction = pred.predict(ref.split(), beam_search=beam_search)
semi.append(prediction + ' <sep> ' + ref)
if count % 100 == 0:
print(count)
write_samples(semi, write_path, 'a')
semi = []
def upload():
global result
global imgname
global idnum
f = request.files['file']
img = Image.open(f.filename)
app = Predict()
result = app.predict(img)
imgname = str(secure_filename(f.filename))
# store to cassandra
global session
session.execute("""INSERT INTO mnist (imgname, result)
VALUES ('""" + imgname + """','""" + str(result) + """')""")
'''session.execute("""INSERT INTO mnist (id, imgname, result) VALUES (1,imgname,result)""")'''
'''session.execute(
"""
INSERT INTO demandtable (id, imgname, result)
VALUES (%s, %s)
""",
(1,imgname,result)
)'''
return
'''
<!doctype html>
<html>
<body> '''+str(result)+'''
def predict():
if request.method == 'POST':
#In case no file was uploaded
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
#In case file has an empty name
if file.filename == '':
flash('No selected file')
return redirect(request.url)
#Everything is correct and we can run the prediction
if file and allowed_file(file.filename):
#save and read uploaded image
filename = secure_filename(file.filename)
file.save(secure_filename(file.filename))
image = Image.open(file.filename)
#flatten_img = np.reshape(image, 784)
app = Predict()
x = app.predict(image)
current_time = str(datetime.datetime.now())
session.execute(
"""
INSERT INTO demandtable (time, filename, result)
VALUES (%s, %s, %s)
""", (current_time, filename, x))
return '识别结果：' + x
class TwoLayerNet:
def __init__(self,
input_size,
hidden_size,
output_size,
weight_init_std=0.01):
n = {}
n["W"] = [
weight_init_std * np.random.randn(s1, s2)
for s1, s2 in ((input_size, hidden_size), (hidden_size,
output_size))
]
n["B"] = [np.zeros(s) for s in (hidden_size, output_size)]
self._net = n
self._p = Predict(sigmoid, softmax)
@property
def params(self):
d = {
"W1": self._net["W"][0],
"W2": self._net["W"][1],
"b1": self._net["B"][0],
"b2": self._net["B"][1]
}
return d
def predict(self, x):
return self._p.predict(self._net, x)
def loss(self, x, t):
y = self.predict(x)
return cross_entropy_error(y, t)
def accuracy(self, x, t):
y = np.argmax(self.predict(x), axis=1)
t = np.argmax(t, axis=1)
return np.sum(y == t) / float(x.shape[0])
def numerical_gradient(self, x, t):
loss_W = lambda W: self.loss(x, t)
return {
i: numerical_gradient(loss_W, d)
for i, d in (("W1", self._net["W"][0]), ("b1", self._net["B"][0]),
("W2", self._net["W"][1]), ("b2", self._net["B"][1]))
}
def on_open_clicked(self, button):
if (not (self.BOX_IMG_TO_PREDICT == None)
and (not (self.IMAGE == None))):
self.box.remove(self.BOX_IMG_TO_PREDICT)
self.BOX_IMG_TO_PREDICT = Gtk.Box()
dialog = Gtk.FileChooserDialog("Open Image", button.get_toplevel(),
Gtk.FileChooserAction.OPEN)
dialog.add_button(Gtk.STOCK_CANCEL, 0)
dialog.add_button(Gtk.STOCK_OPEN, 1)
dialog.set_default_response(1)
filefilter = Gtk.FileFilter()
filefilter.add_pixbuf_formats()
dialog.set_filter(filefilter)
if ((dialog.run() == 1)
and ((dialog.get_filename().endswith(".jpeg") or
(dialog.get_filename().endswith(".jpg"))) == True)):
self.image = Gtk.Image()
self.image.set_from_file(dialog.get_filename())
self.BOX_IMG_TO_PREDICT.add(self.image)
self.BOX_IMG_TO_PREDICT.set_size_request(150, 150)
#CNN:
result = Predict.predict(dialog.get_filename())
if result == 0:
button.set_label(
"I see a CAT, Could you show me a photo (jpeg/jpg) of a cat or a dog? "
)
elif result == 1:
button.set_label(
"I see a DOG, Could you show me a photo (jpeg/jpg) of a cat or a dog?"
)
self.box.add(self.BOX_IMG_TO_PREDICT)
self.window.show_all()
else:
button.set_label(
"It isn't a photo(jpeg/jpg), Could you show me a photo of a cat or a dog?"
)
dialog.destroy()
def predictResult(self, request, context):
if not os.path.isdir('./PredictDir'):
os.makedirs('./PredictDir')
guid = ''
for r in request:
if not os.path.isdir('./PredictDir/{}'.format(r.guid)):
os.makedirs('./PredictDir/{}'.format(r.guid))
if guid == '':
guid = r.guid
with open('./PredictDir/{}/{}'.format(r.guid, r.name), 'ab') as f:
f.write(r.file)
p = Predict()
predictPath = p.predict(guid)
with open(predictPath, 'rb') as f:
lines = f.readlines()
for line in tqdm(lines):
yield basic_pb2.PredictResultFile(file=line)
def test_predict():
for nm in ['nnr_k']:
p = Predict(nm, 'np')
departure_planned = '08:17'
departure_actual = '08:17'
departure_station = 'DIS'
intermediate_actual = '09:02'
intermediate_station = 'COL'
arrival_planned = '10:05'
arrival_actual = '10:11'
arrival_station = 'LST'
for m in range(1, 13):
arrival_predict = p.predict(departure_planned, departure_actual, departure_station,
arrival_planned, arrival_station,
intermediate_actual, intermediate_station,
datetime.datetime(2020, m, 1))
print(arrival_predict)
def predict():
global values
global customer
global percentile
if selected_filename == 'file_not_selected':
st.write("Please select a customer data file to proceed")
return
customer = st.sidebar.selectbox("Select a customer for predicting churn",
("5575-GNVDE", "1452-KIOVK", "9763-GRSKD"))
risk_factor = st.sidebar.slider("How critical retaining the customer is?",
0.0, 1.0, 0.50, 0.1)
percentile = risk_factor
prediction = Predict(train_data, model, customer)
conditioned_sf = prediction.predict()
predictions = prediction.predict_percentile(conditioned_sf, percentile)
values = prediction.predict_remaining_value(predictions, percentile)
def scan():
if request.method == 'POST':
# get scanned image from user
img_name = video_stream.save_frame()
image_path = "/Users/naresh/Documents/University/Hackathons/CUES_Global_Solutions_Hacakathon/lent21/static/images/" + img_name
# render loading screen
#render_template('scan.html', loading=1, feed=0, img_name=img_name)
# predict what topic question is from
Pred = Predict()
search_key = Pred.predict(image_path) # get topic (search_key)
# handle errors
if search_key == "error":
flash('Image was not scanned correctly, please try again!')
return redirect('/scan')
# search youtube for relevant videos
results = youtube_search(search_key, max_results=3)
vid_list = get_video_codes(results)
#print(vid_list)
# find related videos
related_questions = get_related_questions(search_key)
# render final template
return render_template('scan.html',
loading=0,
feed=0,
img_name=img_name,
search_key=search_key,
vid_list=vid_list,
related_questions=related_questions)
return render_template('scan.html',
loading=0,
feed=1,
related_questions=[])
'dataFiles/forecast/request.csv'))
print(data_file_path)
data = pd.read_csv(data_file_path)
if not data.empty:
data.columns = [
"Month", "Year", "Day", "date", "Season", "sales", "ProductType"
]
fdp = ForecastDataProcessing()
data = fdp.aggregate_monthly(data)
diff = fdp.get_diff(data)
lagged = fdp.add_lag(diff, 13)
scaled = fdp.scale_and_reshape(lagged)
model_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'TimeSeriesForecast/models/forecast.h5'))
print("modelpath: ", model_path)
model = Predict(model_path)
output = model.predict(scaled)
parsed_output = fdp.process_output(output, data, scaled)
response_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..',
'dataFiles/forecast/forecast_response.csv'))
parsed_output.to_csv(response_path)
def predict(self, testData, tree):
# Creates a object of Predict class and passes the test data to be predicted.
pr = Predict(tree)
result = pr.predict(testData)
return result
def main():
storage_name = '104'
do_preprocessing = True
train_new_model = True
do_prediction = True
plot_results = True
plot_baseline = False
use_temporal_features = True
passenger_amount = True
test_set_length = 24 * 28
if do_preprocessing:
sequence_length = 24 * 7
path_to_data_folder = 'raw_data/'
difference_length = 24 * 7
use_difference = True
bus_nr = '150'
bus_direction = True
pp = preprocess(storage_name, sequence_length, test_set_length,
path_to_data_folder, difference_length, use_difference,
bus_nr, bus_direction, passenger_amount,
use_temporal_features)
# get and print some preprocessorially defined hyperparameters
utils = Utils(storage_name)
#print(utils.partition['test'])
for key, value in utils.hyper_params.items():
print(f'{key}:\t{value}')
# Parameters
train_params = {'batch_size': 64, 'shuffle': True, 'num_workers': 0}
max_epochs = 4
test_params = {'batch_size': 2, 'shuffle': False, 'num_workers': 0}
n_samples = 32 # number of samples when sampling the predictions
n_std = 2 # number of standard deviations away from the mean to set confidence interval
if use_temporal_features:
input_dimension = 41 # number of features
else:
input_dimension = 10
hidden_dimension = 128 # dimension of hidden lstm layer
linear_dimension = 64 # dimension of output of first linear layer
output_dimension = 10 # output dimension of network
sequence_length = utils.hyper_params['sequence_length']
difference_length = utils.hyper_params['difference_length']
elbo_sample_nbr = 3 # The number of times of the weight-sampling and predictions done to gather the loss
net = LSTM_Net(input_dim=input_dimension,
hidden_dim=hidden_dimension,
linear_dim=linear_dimension,
sequence_length=sequence_length,
output_dim=output_dimension)
#print(net)
stOps = StorageOps(storage_name)
if train_new_model:
trained_model, train_losses, test_losses = train_model(
model=net,
max_epochs=max_epochs,
params=train_params,
partition=utils.partition,
elbo_sample_nbr=elbo_sample_nbr,
)
stOps.store_model(trained_model, train_losses, test_losses,
input_dimension, hidden_dimension, linear_dimension,
output_dimension)
else:
trained_model, train_losses, test_losses, input_dimension, hidden_dimension, linear_dimension, output_dimension = stOps.load_model(
)
print(input_dimension, hidden_dimension, linear_dimension,
output_dimension)
predicting = Predict(name=storage_name,
trained_model=trained_model,
difference_length=difference_length,
test_set_IDs=utils.partition['test'],
params=test_params,
n_samples=n_samples,
n_std=n_std,
passenger_amount=passenger_amount)
if do_prediction:
predicting.predict()
stOps.save_targets_means_stds(predicting.targets, predicting.means,
predicting.stds)
stOps.save_mse_and_accuracy(
predicting.get_MSE(predicting.targets, predicting.means),
predicting.get_accuracy(predicting.targets, predicting.means,
predicting.stds))
else:
targets, means, stds = stOps.load_targets_means_stds()
stOps.save_mse_and_accuracy(
predicting.get_MSE(targets, means),
predicting.get_accuracy(targets, means, stds))
if plot_results:
res = Results(storage_name, passenger_amount)
res.plot_trend_given_time(7)
res.plot_trend_given_stop(0)
res.plot_predictions_all_stops(7)
res.plot_predictions_given_stop(0)
'''for i in range(24):
res.plot_trend_given_time(i)
for i in range(10):
res.plot_trend_given_stop(i)
for i in range(test_set_length):
res.plot_predictions_all_stops(i)
for i in range(10):
res.plot_predictions_given_stop(i)
res.plot_predictions_given_stop_48h(i)
res.plot_training()'''
res.print_performance_measures()
if plot_baseline:
baseline = Baseline(storage_name)
baseline.get_predictions_lower_upper()
baseline.get_mse_and_acc()
for hour in range(24):
baseline.plot_predictions_all_stops(hour)
for stop in range(10):
baseline.plot_predictions_given_stop(stop)
baseline.plot_predictions_given_stop_48h(stop)
print(baseline.mse)
print(baseline.acc)
import argparse
import time
from predict import Predict
from util import display_elapsed_time
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Run CheXpert prediction")
parser.add_argument('input_csv', type=str,
help="path of the input csv file that contains paths of the images to run predictions on")
parser.add_argument('output_csv', type=str,
help="path where to save the csv file that contains predictions for each study")
parser.add_argument('--model_dirs', type=str, metavar='m', default="./models/report_baseline/m1,./models/report_baseline/m2",
help="comma-separated list of dirs containing models to use for inference (default= ./models/report_baseline/m1,./models/report_baseline/m2)")
args = parser.parse_args()
started = time.time()
model_dirs = args.model_dirs.split(',')
model = Predict(args.input_csv, model_dirs=model_dirs, output_csv=args.output_csv)
model.predict()
print("Done")
display_elapsed_time(started)
from predict import Predict
from config import root_path
import json
if __name__ == '__main__':
text = '装帧全部采用日本进口竹尾纸，专为读书人打造奢华手感 ◆ 畅销100万册，独占同名书市场七成份额...'
predict = Predict()
label, score = predict.predict(text)
print('label:{}'.format(label))
print('score:{}'.format(score))
with open(root_path + '/data/label2id.json', 'r') as f:
label2id = json.load(f)
print(list(label2id.keys())[list(label2id.values()).index(label)])
from hypotest import Hypotest
#case 2_1
np.set_printoptions(precision=4)
X = np.array([[-3, 2],
[-2, 1.5],
[-1, 1],
[0, 0.5],
[1, 0]])
y = np.array([[1], [1], [1], [-1], [-1]])
pe=Perceptron(10,X,y)
theta_perceptron, num = pe.perceptron()
sv=Svm(X,y)
theta_svm = sv.svm()
pr=Predict(theta_perceptron,np.array([[1], [-2]]))
print("From preceptron algorithm, the theta is:",theta_perceptron,"using direction [[1], [-2]], it will be predicted as:",pr.predict())
pr2=Predict(theta_svm,np.array([[1], [-2]]))
print("From svm, the theta is:",theta_svm,"using direction [[1], [-2]], it will be predicted as:",pr2.predict())
#case 3_1
np.set_printoptions(precision=4)
X = np.array([[-3, 2],
[-2, 1.5],
[-1, 1],
[0, 0.5],
[1, 0]])
y = np.array([[1], [1], [1], [-1], [-1]])
z = np.array([[1], [-2]])
pl=Probclearn(X,y)
q,mu_pos,mu_neg,sigma2_pos,sigma2_neg = pl.probclearn()
def predict(args):
pred = Predict(path_to_data=args.path_to_overall)
pred.predict(args.tconst, args.display)
class RougeEval():
def __init__(self, path, vocab=None):
self.predict = Predict()
self.rouge = Rouge()
self.testset = SamplesDataset(path, vocab=self.predict.vocab)
print('test vocab: ', len(self.testset.vocab))
self.refs = self.process()
self.hypos = self.build_hypos()
self.scores = self.get_scores()
def process(self):
tgts = self.testset['tgt']
refs = []
for tgt in tgts:
ref = ' '.join(tgt).replace('。', '.')
refs.append(ref)
print(f'Test set contains {len(self.testset)} samples.')
return refs
@timer('building hypotheses')
def build_hypos(self):
"""Generate hypos for the dataset.
Args:
predict (predict.Predict()): The predictor instance.
"""
print('Building hypotheses.')
count = 0
exception_count = 0
hypos = []
for sample in self.testset:
# print(' '.join(sample['source']))
count += 1
if count % 100 == 0:
print(count)
# try:
hypo = self.predict.predict(sample['x'], sample['OOV'], sample['img_vec'])
hypos.append(hypo)
# except Exception as e:
# print(sample)
# print(e)
# exit(0)
# try:
# self.hypos.append(predict.predict(sample['x'], sample['OOV'], sample['img_vec']))
# except:
# exception_count += 1
# print('exception: ', exception_count)
# self.hypos.append('')
return hypos
def get_scores(self):
assert len(self.hypos) > 0, 'Build hypotheses first!'
print('Calculating average rouge scores.')
for hypo, ref in zip(self.hypos, self.refs):
print(hypo, ' ||| ', ref)
return self.rouge.get_scores(self.hypos, self.refs, avg=True)
def one_sample(self, hypo, ref):
return self.rouge.get_scores(hypo, ref)[0]
def get_metric(self, metric):
return self.scores[metric]['p']
class ClassifierRun():
def __init__(self, cfg_file_name):
self.config = ConfigParser.ConfigParser()
self.cur_dir = os.path.dirname(os.path.abspath(cfg_file_name))
self.cfg_parser(cfg_file_name)
self.preprocess = Preprocess(cfg_file_name)
self.cnt = 0
self.train_features = []
self.train_labels = []
self.test_features = []
self.test_labels = []
self.test_names = []
self._train = Train(self.space, self.params)
self._predict = Predict()
self._rule = Rule()
self._tree = ClassTreePedict('./resource/cate_id.cfg', './model')
def cfg_parser(self, cfg_file_name):
self.config.read(cfg_file_name)
section = 'model'
if self.config.has_option(section, 'model_file'):
self.model_path = self.config.get(section, 'model_file')
else:
self.model_path = './model/testmodel'
if self.config.has_option(section, 'model_dir'):
self.model_dir = self.config.get(section, 'model_dir')
else:
self.model_dir = './model'
if self.config.has_option(section, 'vec_space') and self.config.get(section, 'vec_space') == 'topic':
self.space = 'topic'
else:
self.space = 'word'
if self.space == 'topic':
if self.config.has_section('topic_param'):
self.params = dict(self.config.items('topic_param'))
elif self.space == 'word':
if self.config.has_section('word_param'):
self.params = dict(self.config.items('word_param'))
section = 'evaluation'
self.test_size_prob = 1.0
if self.config.has_option(section, 'test_size'):
self.test_size_prob = self.config.getfloat(section, 'test_size')
if self.config.has_option(section, 'random_state'):
seed = self.config.getint(section, 'random_state')
random.seed(seed)
self.level = 0
section = 'default'
if self.config.has_option(section, 'level'):
self.level = self.config.getint(section, 'level')
if self.config.has_option(section, 'cate_id_file'):
self.cate_id_file = self.config.get(section, 'cate_id_file')
else:
self.cate_id_file = "resource/cate_id.cfg"
logging.info('[Done] config parsing')
logging.info('use %s space, params=%s' %(self.space, json.dumps(self.params) ))
def train(self):
self._train.train(self.train_features, self.train_labels)
self._train.dump_model(self.model_path)
def test(self):
if self.model_path.endswith('rule'):
self._rule.load_rule(self.model_path)
is_rule = True
else:
self._predict.load_model(self.model_path)
is_rule = False
print len(self.test_features)
for (features, label, name) in zip(self.test_features, self.test_labels, self.test_names):
if is_rule:
result = self._rule.predict(name, 0)
else:
result = self._predict.predict(features)
print result.encode('u8'),'\t', label.encode('u8'),'\t', name.encode('u8'), '\t', json.dumps(features,'\t', ensure_ascii=False).encode('u8')
def testone(self, name, cat_name, brand, price):
tree = ClassTreePedict(self.cate_id_file, model_dir)
features = self.preprocess.process(name, cat_name, brand, price, level=0)
features = json.loads('{"Eden": 1, "Botkier": 1, "Satchel": 1, "马毛": 1, "女士": 1, "柏柯尔": 1, "拼接": 1, "手提包": 1, "Small": 1 }')
result = tree.predict(name, features, indexclass=u"root")
print result.encode('u8'), name.encode('u8'), json.dumps(features, ensure_ascii=False).encode('u8')
# map_cfg 类目和ID的映射文件，model_dir 存放模型文件目录，data_file 数据文件
def predict(self, map_cfg, model_dir, data_file_name):
tree = ClassTreePedict(map_cfg, model_dir)
data_file = open(data_file_name, 'r')
for line in data_file:
line = line.strip()
try:
old_cate, cid_cate, name, brand, price = line.decode('u8').split(u'\t')
except Exception,e :
print >> sys.stderr, "Error:", line
print >> sys.stderr, e
sys.exit()
cat_name = json.dumps(cid_cate.split(','))
price = float(price)
features = self.preprocess.process(name, cat_name, brand, price, level=0)
#result = tree.predict(name, features, indexclass=u"root")
indexclass = u'root'
result = tree.predict(name, features, indexclass, price, cat_name)
print "%s\t%s\t%s" %(result.encode('u8'), old_cate.encode('u8'), name.encode('u8'))
data_file.close()
train_feature = hog_train.HoG_output_vector()
print("\n >>>>>> Extract Train Feature !!!!!! Suceess !!!!!! \n")
hog_test = extract_feature(dir_save_feature, "X_test", X_test[0:1],
block_size, block_stride, cell_size, bin_num)
test_feature = hog_test.HoG_output_vector()
print("\n >>>>>> Extract Test Feature !!!!!! Suceess !!!!!! \n")
elif task_type == 'result':
############## (c)Fit Model & Predict
TPR = []
FPR = []
for t in range(10, 37):
thresh = t / 50
Pre = Predict(train_feature, test_feature, y_train, y_test, thresh)
train_predict, test_predict = Pre.predict()
########################### Step 3 : Analysis ##############################
#ROC曲线
true_pos = sum((y_test == 1) * test_predict)
true_neg = sum((y_test == 0) * (test_predict == 0))
false_pos = sum((y_test == 0) * (test_predict == 1))
false_neg = sum((y_test == 1) * (test_predict == 0))
TPR.append(true_pos / (false_neg + true_pos))
FPR.append(false_pos / (false_pos + true_neg)) # False Positive Rate
plt.figure()
plt.plot(FPR, TPR, 'r--', 5)
plt.title('ROC Curve')
plt.xlabel('False Positive Rate')
'.')[0] + '_result.txt'
fin = open(image_list_file, 'r')
fout = open(result_file_name, 'w')
resize_scale = 1.0
score_threshold = 0.11
NMS_threshold = 0.4
counter = 0
for line in fin:
line = line.strip('\n')
im = cv2.imread(os.path.join(fddb_image_root, line + '.jpg'),
cv2.IMREAD_COLOR)
bboxes = my_predictor.predict(im,
resize_scale=resize_scale,
score_threshold=score_threshold,
top_k=10000,
NMS_threshold=NMS_threshold)
# for bbox in bboxes:
# cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 255, 0), 1)
# cv2.imshow('im', im)
# cv2.waitKey()
fout.write(line + '\n')
fout.write(str(len(bboxes)) + '\n')
for bbox in bboxes:
fout.write('%d %d %d %d %.03f' %
(math.floor(bbox[0]), math.floor(bbox[1]),
math.ceil(bbox[2] - bbox[0]), math.ceil(bbox[3] - bbox[1]),
bbox[4] if bbox[4] <= 1 else 1) + '\n')
def evaluation(self, model_path, eval_file, predict_file, badcase_file):
print "%d samples are used to train, %d samples are used to test" % (
len(self.train_features), len(self.test_features))
_predict = Predict()
_predict.load_model(model_path)
#report = classification_report(self.test_labels, self._predict.predict(self.test_features))
efile = open(eval_file, 'w+')
pfile = open(predict_file, 'w')
bfile = open(badcase_file, 'w')
cate_cnt = dict()
accuracy_cnt = 0
all_cnt = 0
print "******************"
for features, label, line in zip(self.test_features, self.test_labels,
self.test_lines):
result = _predict.predict(features)
#print result.encode('u8'), label.encode('u8'), name.encode('u8')
all_cnt += 1
if all_cnt % 1000 == 0:
print >> sys.stderr, "test %d samples ... " % all_cnt
cate_key = label
pred_key = result
cate_key = cate_key.replace('#', '')
pred_key = pred_key.replace('#', '')
if cate_key not in cate_cnt:
cate_cnt[cate_key] = {'cate': 0, 'pred': 0, 'corr': 0}
if pred_key not in cate_cnt:
cate_cnt[pred_key] = {'cate': 0, 'pred': 0, 'corr': 0}
cate_cnt[cate_key]['cate'] += 1
cate_cnt[pred_key]['pred'] += 1
print >> pfile, "%s\t%s" % (result.encode('u8'), line)
if cate_key == pred_key:
cate_cnt[cate_key]['corr'] += 1
accuracy_cnt += 1
else:
print >> bfile, "%s\t%s\t%s" % (
result.encode('u8'), line,
json.dumps(features, ensure_ascii=False).encode('u8'))
print >> efile, "key\tcate_num\tpred_num\tcorrect_num\tprecision\trecall\tF_score"
F_score_cnt = 0
for key, vdic in cate_cnt.iteritems():
cate_num = vdic['cate']
pred_num = vdic['pred']
corr_num = vdic['corr']
if pred_num == 0 or cate_num == 0:
print >> efile, "%s\t%d\t%d\t%d" % (
key.encode('utf-8'), cate_num, pred_num, corr_num)
else:
precision = corr_num * 1.0 / pred_num
recall = corr_num * 1.0 / cate_num
F_score = 0.0
if (precision + recall) > 1e-3:
F_score = 2 * precision * recall / (precision + recall)
print >> efile, "%s\t%d\t%d\t%d\t%.3f\t%.3f\t%.3f" % (
key.encode('utf-8'), cate_num, pred_num, corr_num,
precision, recall, F_score)
print >> efile, "accuracy: ", accuracy_cnt * 1.0 / all_cnt
print >> efile, "%d samples are used to train, %d samples are used to test" % (
len(self.train_features), len(self.test_features))
efile.close()
pfile.close()
bfile.close()
import time
import cv2
import numpy as np
from load_model import load_flattened_model, load_model
from predict import Predict
from canvas import canvas_with_skeleton
# model = load_flattened_model()
model = load_model()
cap = cv2.VideoCapture(0)
while (True):
_, img = cap.read()
t = time.time()
keypoints = Predict.predict(model, img)
t = time.time() - t
canvas = canvas_with_skeleton(img, keypoints)
canvas = cv2.resize(canvas, None, fx=2, fy=2)
cv2.putText(canvas, 'fps: %s' % (1 / t), (60, 60),
cv2.FONT_HERSHEY_SIMPLEX, 2, 255)
cv2.imshow('frame', canvas)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()