def setupUi(self, MainWindow):
MainWindow.setObjectName("Homework")
MainWindow.resize(414, 285)
self.centralWidget = QtWidgets.QWidget(MainWindow)
self.centralWidget.setObjectName("centralWidget")
self.scoreButton = QtWidgets.QPushButton(self.centralWidget)
self.scoreButton.setGeometry(QtCore.QRect(280, 120, 89, 25))
self.scoreButton.setObjectName("scoreButton")
self.resultLabel = QtWidgets.QLabel(self.centralWidget)
self.resultLabel.setGeometry(QtCore.QRect(180, 203, 64, 64))
self.resultLabel.setObjectName("resultLabel")
self.widget = QtWidgets.QWidget(self.centralWidget)
self.widget.setGeometry(QtCore.QRect(30, 30, 201, 171))
self.widget.setObjectName("widget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.widget)
self.verticalLayout.setContentsMargins(11, 11, 11, 11)
self.verticalLayout.setSpacing(6)
self.verticalLayout.setObjectName("verticalLayout")
self.label = QtWidgets.QLabel(self.widget)
self.label.setObjectName("label")
self.verticalLayout.addWidget(self.label)
self.tweetTextEdit = QtWidgets.QPlainTextEdit(self.widget)
self.tweetTextEdit.setObjectName("tweetTextEdit")
self.verticalLayout.addWidget(self.tweetTextEdit)
MainWindow.setCentralWidget(self.centralWidget)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
# non-ui
self.setupEvents()
self.predictor = Predictor("models/predictor.pkl")
def send_file():
global predictor
if predictor is None:
predictor = Predictor()
result = ''
if 'file2upload' not in request.files:
print('No file part')
return redirect(request.url)
fileob = request.files["file2upload"]
if fileob.filename == '':
print('No selected file')
return redirect(request.url)
if fileob and allowed_file(fileob.filename):
filename = secure_filename(fileob.filename)
#result = predictor.predict(fileob)
#Once we want to store the files on the server
save_path = os.path.join(app.config["UPLOAD_FOLDER"], filename)
fileob.save(save_path)
with open(save_path, "r") as f:
result = predictor.predict(f)
#try:
# os.remove(save_path)
#except Exception as error:
# app.logger.error("Error deleting file after processing", error)
return result
def update_prediction_convergence(static, inputs, models):
if (inputs is not None) and (models is not None):
# Filter inputs
file = models
inputs = sorted(inputs)
# Make filename
for parameter in inputs:
temp = parameter.strip(" ")
file += "_" + temp
# Initialise model
model = Predictor(dataset, inputs=inputs, model=models, filename=file)
# Run
results = model.check()
return {
'data': [{
'x': results[-1],
'y': results[1],
'line': dict(color='black')
}]
}
def main(filename):
#Loading data
dataset = DataLoader(filename)
explorer = DataExplorer(dataset)
predictor = Predictor(dataset)
#Exploring data
explorer.visualise_distributions(without_outliers=False)
explorer.visualise_distributions(without_outliers=True,
identify_abnormal=True)
explorer.describe_variable(652)
explorer.histogram(652, without_outliers=True)
explorer.boxplot(652, without_outliers=True)
#PCA
explorer.pca(plot=True)
#Correlations
explorer.best_relationship_class()
explorer.visualise_best_relationship_class()
explorer.best_relationship_pca()
explorer.visualise_best_relationship_pca()
#SVM
a = Predictor(dataset)
a.svc()
def callback(self, ch, method, properties, body):
print(" [x] Received %r" % body)
'''
model predict code here.
write to csv
'''
predictor = Predictor()
face_img_path = body
if not os.path.exists(face_img_path):
print("face path not exist.")
return
img_name = str(body).split('\\')[-1]
# print(img_name)
with open(face_img_path, 'rb') as f:
image_bytes = f.read()
print("inference:")
start_time = time.time()
label = predictor.predict(image_bytes, img_name)
end_time = time.time()
inference_latency = end_time - start_time
print("label = " + str(label) + ", " + str(inference_latency))
print(" Run model predict here. \n")
def dump():
predictor = Predictor()
tf_word_feature = predictor.forward_allword()
sess = melt.load(FLAGS.model_dir)
word_feature = sess.run(tf_word_feature)
np.save(FLAGS.data_dir + '/word_feature.npy', word_feature)
sess.close()
def index():
learnfile = request.files.get('learnfile')
testfile = request.files.get('testfile')
if learnfile is None or testfile is None:
return template('index.html', status='Загрузите файлы', results=[])
name_learnfile, ext_learnfile = os.path.splitext(learnfile.filename)
name_testfile, ext_testfile = os.path.splitext(testfile.filename)
if ext_learnfile not in ('.zip',) or ext_testfile not in ('.zip',):
return template('index.html', status='Не то расширение файла. Нужен zip', results=[])
current_id = next(id_gen())
learn_zip = os.path.join(WORK_DIR, 'learn-{0}.zip'.format(current_id))
test_zip = os.path.join(WORK_DIR, 'test-{0}.zip'.format(current_id))
learn_dir = os.path.join(WORK_DIR, 'learn-{0}'.format(current_id))
test_dir = os.path.join(WORK_DIR, 'test-{0}'.format(current_id))
learnfile.save(learn_zip)
testfile.save(test_zip)
unzip_status_learn = unzip_file(learn_zip, learn_dir)
unzip_status_test = unzip_file(test_zip, test_dir)
if not os.path.exists(learn_dir) or not os.path.exists(test_dir):
return template('index.html', status='Ошибка распаковки', results=[])
predictor = Predictor(learn_dir, test_dir)
accuracy, results = predictor.estimate()
return template('index.html', status='Держи ответ:', accuracy= accuracy, results=results)
def infer(self, args: ClassifierArgs) -> Dict:
content = args.content
assert content is not None, 'in infer mode, parameter content cannot be None! '
content = content.strip()
assert content != '' and len(
content) != 0, 'in infer mode, parameter content cannot be empty! '
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
pred_probs = predictor.predict(content)
pred_label = np.argmax(pred_probs)
pred_label = self.data_reader.get_idx_to_label(pred_label)
if pred_label == '100':
pred_label = '0'
elif pred_label == '101':
pred_label = '1'
result_in_dict = {
'content': content,
'pred_label': pred_label,
'pred_confidence': pred_probs
}
result_in_str = ', '.join([
'{}: {}'.format(key, value) if not isinstance(value, list) else
'{}: [{}]'.format(key, ', '.join(["%.4f" % val for val in value]))
for key, value in result_in_dict.items()
])
print(result_in_str)
logging.info(result_in_str)
return result_in_dict
def eval_model(args) -> None:
predictor = Predictor(args.archive_file, args.cuda_device,
args.predicted_pages, args.merge_google,
args.score_format, args.verbose)
raw_data = []
with open(args.in_file) as f:
for line in f:
raw_data.append(json.loads(line))
actual = []
predicted = []
if args.log is not None:
f = open(args.log, "w+")
#print({util.get_device_of(param) for param in model.parameters()})
for output in predictor.predict(raw_data[args.start:args.end]):
actual.append(output['actual'] if 'actual' in
output else output.get('label', 'NOT ENOUGH INFO'))
predicted.append(output['predicted_label'] if 'predicted_label' in
output else output['predicted'])
if args.log is not None:
f.write(json.dumps(output) + "\n")
if args.log is not None:
f.close()
if args.verbose:
print(accuracy_score(actual, predicted))
print(classification_report(actual, predicted))
print(confusion_matrix(actual, predicted))
def validate(epoch_idx, abstracts1):
model.load_state_dict(torch.load(args.save))
print("model restored")
test_loader = DataLoader(abstracts1, config.batch_size)
eval_f = Evaluate()
num_exams = 3
predictor = Predictor(model, abstracts1.vectorizer)
print("Start Evaluating")
print("Test Data: ", len(abstracts1))
cand, ref = predictor.preeval_batch(test_loader, len(abstracts1),
num_exams)
scores = []
final = []
fields = ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4", "METEOR", "ROUGE_L"]
for i in range(6):
scores.append([])
for i in range(num_exams):
print("No.", i)
final_scores = eval_f.evaluate(live=True, cand=cand[i], ref=ref)
for j in range(6):
scores[j].append(final_scores[fields[j]])
with open('figure.pkl', 'wb') as f:
pickle.dump((fields, scores), f)
# Start writing ...
f_out_name = cwd + config.relative_score_path
f_out_name = f_out_name % epoch_idx
f_out = open(f_out_name, 'w')
for j in range(6):
f_out.write(fields[j] + ': ')
f_out.write(str(final_scores[fields[j]]) + '\n')
final.append(final_scores[fields[j]])
f_out.close()
print("FFFF = ", final_scores)
return sum(final) / float(len(final))
def get_test_prediction(data_value_arrays, labels, model):
p = Predictor(model,
file='/tmp/predict_gbm.log',
loglevel=logging.DEBUG)
y_predicted_df = p.predict(data_value_arrays, labels)
rmse = p.score(data_value_arrays, labels)
return rmse, y_predicted_df
def predict_tips():
# Validate the access token
authorization = request.headers.get('Authorization', None)
if authorization is None or authorization.split(' ')[1] != os.environ.get(
'ACCESS_TOKEN'):
return {'error': 'Please provide a valid access token'}, 400
# Retrieve the request body
req_body = request.get_json()
if req_body is None:
return {'error': 'Please provide necessary parameters'}, 400
try:
# Parse the request body to retreive required parameters
user_id, search_dates = parse_request_body(req_body)
except Exception as err:
print(err)
return {'error': str(err)}, 400
try:
# Get predicted tips for each specified day
predictor = Predictor(user_id=user_id, search_dates=search_dates)
return predictor.get_predicted_tips(), 200
except DatasetError as err:
return {'error': str(err)}, 500
except Exception as err:
print(err)
return {
'error':
'Failed to predict tips for the user. Please try again later'
}, 500
def evaluate():
reader = PWKPReader()
vocab = Vocabulary.from_files(vocab_dir)
iterator = BasicIterator(batch_size=opt.batch_size)
iterator.index_with(vocab)
model = Seq2Seq(emb_size=opt.emb_size,
hidden_size=opt.hidden_size,
enc_layers=opt.enc_layers,
dec_layers=opt.dec_layers,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
beam_size=opt.beam_size,
label_smoothing=opt.label_smoothing,
vocab=vocab)
model = model.cuda(opt.gpu)
model_state = torch.load(opt.restore, map_location=util.device_mapping(-1))
model.load_state_dict(model_state)
predictor = Predictor(iterator=iterator,
max_decoding_step=opt.max_step,
vocab=vocab,
reader=reader,
data_path=test_path,
log_dir=save_dir,
map_path=ner_path,
cuda_device=opt.gpu)
predictor.evaluate(model)
def train_epoches(t_dataset, v_dataset, model, n_epochs,
teacher_forcing_ratio):
eval_f = Evaluate_test()
best_dev = 0
train_loader = t_dataset.corpus
len_batch = len(train_loader)
epoch_examples_total = t_dataset.len
for epoch in range(1, n_epochs + 1):
model.train(True)
torch.set_grad_enabled(True)
epoch_loss = 0
for batch_idx in range(len_batch):
loss, num_examples = train_batch(t_dataset, batch_idx, model,
teacher_forcing_ratio)
epoch_loss += loss * num_examples
sys.stdout.write('%d batches processed. current batch loss: %f\r' %
(batch_idx, loss))
sys.stdout.flush()
epoch_loss /= epoch_examples_total
log_msg = "Finished epoch %d with losses: %.4f" % (epoch, epoch_loss)
print(log_msg)
predictor = Predictor(model, v_dataset.vocab, args.cuda)
print("Start Evaluating")
cand, ref = predictor.preeval_batch(v_dataset)
print('Result:')
print('ref: ', ref[1][0])
print('cand: ', cand[1])
final_scores = eval_f.evaluate(live=True, cand=cand, ref=ref)
epoch_score = 2 * final_scores['ROUGE_L'] * final_scores['Bleu_4'] / (
final_scores['Bleu_4'] + final_scores['ROUGE_L'])
if epoch_score > best_dev:
torch.save(model.state_dict(), args.save)
print("model saved")
best_dev = epoch_score
def test(cf, logger, max_fold=None):
"""performs testing for a given fold (or held out set). saves stats in evaluator.
"""
logger.time("test_fold")
logger.info('starting testing model of fold {} in exp {}'.format(
cf.fold, cf.exp_dir))
net = model.net(cf, logger).cuda()
batch_gen = data_loader.get_test_generator(cf, logger)
test_predictor = Predictor(cf, net, logger, mode='test')
test_results_list = test_predictor.predict_test_set(
batch_gen,
return_results=not hasattr(cf, "eval_test_separately")
or not cf.eval_test_separately)
if test_results_list is not None:
test_evaluator = Evaluator(cf, logger, mode='test')
test_evaluator.evaluate_predictions(test_results_list)
test_evaluator.score_test_df(max_fold=max_fold)
mins, secs = divmod(logger.get_time("test_fold"), 60)
h, mins = divmod(mins, 60)
t = "{:d}h:{:02d}m:{:02d}s".format(int(h), int(mins), int(secs))
logger.info('Testing of fold {} took {}.'.format(cf.fold, t))
def experiment_1():
"""
This experiment tries to predict the values of IVOL from the data by directly looking at the raw variables.
"""
# Instance from Predictor Class
p = Predictor(len(experiment_features), 32, len(target_label),
1).to(device)
# Stochastic Gradient Descent optimizer
optimizer = torch.optim.SGD(p.parameters(),
lr=lr,
momentum=learning_momentum)
# Temporary variables for computing the average loss
all_loss = 0.0
all_count = 0
# range from 1968 to 2019
dt = tqdm(
get_stock_data(experiment_train_start_year, experiment_train_end_year))
optimizer.zero_grad()
for X, Y in dt:
# Prediction
_, loss = p(X, Y)
all_loss += loss.item()
all_count += 1
loss.backward()
if all_count and all_count % cumulate_loss == 0:
nn.utils.clip_grad_norm_(p.parameters(), max_grad_norm)
optimizer.step()
optimizer.zero_grad()
dt.set_description("Average Loss: {:.2f}".format(all_loss / all_count))
def index():
learnfile = request.files.get('learnfile')
testfile = request.files.get('testfile')
if learnfile is None or testfile is None:
return template('index.html', status='Загрузите файлы', results=[])
name_learnfile, ext_learnfile = os.path.splitext(learnfile.filename)
name_testfile, ext_testfile = os.path.splitext(testfile.filename)
if ext_learnfile not in ('.zip', ) or ext_testfile not in ('.zip', ):
return template('index.html',
status='Не то расширение файла. Нужен zip',
results=[])
current_id = next(id_gen())
learn_zip = os.path.join(WORK_DIR, 'learn-{0}.zip'.format(current_id))
test_zip = os.path.join(WORK_DIR, 'test-{0}.zip'.format(current_id))
learn_dir = os.path.join(WORK_DIR, 'learn-{0}'.format(current_id))
test_dir = os.path.join(WORK_DIR, 'test-{0}'.format(current_id))
learnfile.save(learn_zip)
testfile.save(test_zip)
unzip_status_learn = unzip_file(learn_zip, learn_dir)
unzip_status_test = unzip_file(test_zip, test_dir)
if not os.path.exists(learn_dir) or not os.path.exists(test_dir):
return template('index.html', status='Ошибка распаковки', results=[])
predictor = Predictor(learn_dir, test_dir)
accuracy, results = predictor.estimate()
return template('index.html',
status='Держи ответ:',
accuracy=accuracy,
results=results)
def evaluate(self, archive):
self.sparseness = None
if self.misclass is None:
self.member1.predicted_label, self.member1.P_class, self.member1.P_notclass = \
Predictor.predict(self.member1.purified)
self.member2.predicted_label, self.member2.P_class, self.member2.P_notclass = \
Predictor.predict(self.member2.purified)
# Calculate fitness function 2
self.misclass = evaluator.evaluate_ff2(self.member1.P_class,
self.member1.P_notclass,
self.member2.P_class,
self.member2.P_notclass)
if self.distance is None:
# Calculate fitness function 1
self.distance = evaluator.evaluate_ff1(self.member1.purified,
self.member2.purified)
# Recalculate sparseness at each iteration
self.sparseness = evaluator.evaluate_sparseness(self, archive)
if self.sparseness == 0.0:
print(self.sparseness)
print("BUG")
self.aggregate_ff = evaluator.evaluate_aggregate_ff(self.sparseness, self.distance)
return self.aggregate_ff, self.misclass
def print_archive(archive):
path = RESULTS_PATH+'/archive'
dst = path + '/'
if not exists(dst):
makedirs(dst)
for i, ind in enumerate(archive):
filename1 = dst + basename(
'archived_' + str(i) + '_mem1_l_' + str(ind.member1.predicted_label) + '_seed_' + str(ind.seed))
plt.imsave(filename1, ind.member1.purified.reshape(28, 28), cmap=cm.gray, format='png')
loaded_label = (Predictor.predict(ind.member1.purified))
assert (ind.member1.predicted_label == loaded_label[0])
assert (ind.member1.predicted_label == Predictor.model.predict_classes(ind.member1.purified))
np.save(filename1, ind.member1.purified)
loaded_label = Predictor.predict(np.load(filename1 + '.npy'))[0]
assert (ind.member1.predicted_label == loaded_label)
assert (np.array_equal(ind.member1.purified, np.load(filename1 + '.npy')))
filename2 = dst + basename(
'archived_' + str(i) + '_mem2_l_' + str(ind.member2.predicted_label) + '_seed_' + str(ind.seed))
plt.imsave(filename2, ind.member2.purified.reshape(28, 28), cmap=cm.gray, format='png')
loaded_label = (Predictor.predict(ind.member2.purified))
assert (ind.member2.predicted_label == loaded_label[0])
assert (ind.member2.predicted_label == Predictor.model.predict_classes(ind.member2.purified))
np.save(filename2, ind.member2.purified)
loaded_label = Predictor.predict(np.load(filename2 + '.npy'))[0]
assert (ind.member2.predicted_label == loaded_label)
assert (np.array_equal(ind.member2.purified, np.load(filename2 + '.npy')))
def process_data(self):
# Clean and transform the text data
cleansed_data = self.pre_process_text_cleanse_transform(self.data)
# vectorize the data
vectorized_data = ett_t.perform_model_transformation(
self.models["vector_model"], cleansed_data)
# Normalize teh data
normalized_data = ett_t.perform_model_transformation(
self.models["normalizar_model"], vectorized_data)
# Get the predciton labels
labelled_data = ett_p.perform_model_predictions(
self.models["prediction_model"], normalized_data)
result_label = pd.DataFrame(labelled_data)
# Get the probabilities dataframe
probabilities_data = ett_p.perform_model_prob_predictions(
self.models["prediction_model"], normalized_data)
result_prob = pd.DataFrame(probabilities_data)
# get the probability of prediction
result_prob['max_value'] = result_prob.max(axis=1)
# A list of dataframe that containes all columns you want to show in UI
result_list = [result_label, result_prob['max_value']]
results = ett_t.combine_dataframe(result_list, 1)
results.columns = ['code', 'proba']
return results
def test(logger):
"""
perform testing for a given fold (or hold out set). save stats in evaluator.
"""
logger.info('starting testing model of fold {} in exp {}'.format(
cf.fold, cf.exp_dir))
net = model.net(cf, logger).cuda()
test_predictor = Predictor(cf, net, logger, mode='test')
test_evaluator = Evaluator(cf, logger, mode='test')
batch_gen = data_loader.get_test_generator(cf, logger)
test_results_list, test_results_list_mask, test_results_list_seg, test_results_list_fusion, testing_epoch = test_predictor.predict_test_set(
batch_gen, cf, return_results=True)
count = test_evaluator.evaluate_predictions(test_results_list,
testing_epoch,
cf,
pth=cf.test_dir,
flag='test')
print('tp_patient {}, tp_roi {}, fp_roi {}'.format(count[0], count[1],
count[2]))
save_test_image(test_results_list, test_results_list_mask,
test_results_list_seg, test_results_list_fusion,
testing_epoch, cf, cf.plot_dir)
def predict(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
assert isinstance(dataset, ListDataset)
if args.predict_numbers == -1:
predict_dataset = dataset.data
else:
predict_dataset = np.random.choice(dataset.data,
size=(args.predict_numbers, ),
replace=False)
description = tqdm(predict_dataset)
metric = RandomSmoothAccuracyMetrics()
for data in description:
tmp_instances = self.mask_instance_decorator(
args, data, args.predict_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
target = self.data_reader.get_label_to_idx(data.label)
pred = predict(tmp_probs, args.alpha)
metric(pred, target)
description.set_description(metric.__str__())
print(metric)
logging.info(metric)
def predictorAPI():
model = request.args.get('model')
data = request.args.get('data')
preObj = Predictor()
prediction = preObj.predict(model, data)
return str(prediction)
def mbti_test():
data = request.get_json(force=True)
text = data['text']
p = Predictor()
mbti = p.run(text)
return jsonify(results=mbti)
class TextRecognition(object):
def __init__(self, path_to_checkpoint):
self.config = self.load_config(path_to_checkpoint)
self.detector = Predictor(self.config)
def load_config(self, path_to_checkpoint):
url_base = '1xiw7ZnT3WH_9HXoGpLbhW-m2Sm2nlthi'
url_config_vgg_transformers = '1TF8effeufpgkHqQFlmNWKsQtCMfDiooa'
# load base config
if os.path.isfile('./config_text_recognition/base.yml'):
base_config = self.read_from_config(
file_yml='./config_text_recognition/base.yml')
else:
base_config = self.download_config(url_base)
# load vgg transformer config
if os.path.isfile('./config_text_recognition/vgg-transformer.yml'):
config = self.read_from_config(
file_yml='./config_text_recognition/vgg-transformer.yml')
else:
config = self.download_config(url_config_vgg_transformers)
# update base config
base_config.update(config)
# load model from checkpoint
base_config['weights'] = path_to_checkpoint
base_config['device'] = 'cpu'
base_config['predictor']['beamsearch'] = False
return base_config
@staticmethod
def download_config(url_id):
url = 'https://drive.google.com/uc?id={}'.format(url_id)
output = gdown.download(url, quiet=True)
with open(output, encoding='utf-8') as f:
config = yaml.safe_load(f)
return config
@staticmethod
def read_from_config(file_yml):
with open(file_yml, encoding='utf-8') as f:
config = yaml.safe_load(f)
return config
def predict(self, image):
image = Image.fromarray(image)
result = self.detector.predict(image)
return result
def predict_on_batch(self, batch_images):
return self.detector.batch_predict(batch_images)
def evaluate():
if opt.part == 'table2pivot':
corpus = Table2PivotCorpus(vocab_size=opt.vocab_size,
max_len=opt.src_max_len,
batch_size=opt.batch_size,
log_dir=opt.dir,
scale=opt.scale,
mode=opt.mode)
else:
corpus = Pivot2TextCorpus(vocab_size=opt.vocab_size,
src_max_len=opt.src_max_len,
tgt_max_len=opt.tgt_max_len,
batch_size=opt.batch_size,
share=opt.share,
log_dir=opt.dir,
scale=opt.scale,
append_rate=opt.append_rate,
drop_rate=opt.drop_rate,
blank_rate=opt.blank_rate,
setting=opt.setting,
mode=opt.mode,
use_feature=opt.feature)
model = Pivot(emb_size=opt.emb_size,
key_emb_size=opt.key_emb_size,
pos_emb_size=opt.pos_emb_size,
hidden_size=opt.hidden_size,
n_hidden=opt.n_hidden,
n_block=opt.n_block,
ff_size=opt.ff_size,
n_head=opt.n_head,
enc_layers=opt.enc_layers,
dec_layers=opt.dec_layers,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
beam_size=opt.beam_size,
max_decoding_step=opt.max_step,
minimum_length=opt.minimum_length,
label_smoothing=opt.label_smoothing,
share=opt.share,
part=opt.part,
vocab=corpus.vocab,
use_feature=opt.feature,
arch=opt.arch)
if opt.fp16:
model.half()
model = model.cuda(opt.gpu)
model_state = torch.load(opt.restore, map_location=util.device_mapping(-1))
model.load_state_dict(model_state)
predictor = Predictor(dataset=corpus.test_dataset,
dataloader=corpus.test_loader,
corpus=corpus,
cuda_device=opt.gpu)
predictor.evaluate(model)
def get_predictors_with_calibration_file(calibration_file_name):
models = get_models_with_calibration_file(calibration_file_name)
return {
"x_offsets_predictor_x_input_only": Predictor(models["x_offsets_model_x_input_only"]),
"x_offsets_predictor": Predictor(models["x_offsets_model"]),
"y_offsets_predictor_y_input_only": Predictor(models["y_offsets_model_y_input_only"]),
"y_offsets_predictor": Predictor(models["y_offsets_model"]),
"angle_offsets_predictor": Predictor(models["angle_offsets_model"])
}
def predict(self, input_data, prediction_year=None):
predictor = Predictor(self.model, self.device)
# Use midpoints of MultiSurv output intervals
midpoints = self.output_intervals
midpoints[1:] = midpoints[1:] - np.diff(midpoints)[0] / 2
prediction = predictor.predict(input_data, prediction_year, midpoints)
feature_representations, risk = prediction
return feature_representations, risk
def CensorComment(self, request, context):
reqComment = request.clientComment # 들어온 채팅 데이터
resCensor = False
predic = Predictor(reqComment)
score = predic.predict_comment()
# 점수 값이 0.51 보다 높으면 악플로 판정/ 낮으면 클린한 글
if score >= 0.51:
resCensor = True # censor comment
return comment_pb2.CensorReply(serverCensor=resCensor)
def call_for_feature(feature,
model,
tokenizer,
couchdb: CouchDB,
redis,
backfill=False):
response = {"meta": {"next_token": None}}
start_time, end_time = get_time_interval(feature, backfill)
page = 0
while "next_token" in response["meta"] and page < 10:
next_token = response["meta"]["next_token"]
print("Getting tweets for %s between %s and %s%s" %
(feature["name"], start_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
end_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
", next_token: %s..." % next_token if next_token else ""))
page += 1
# Get tweets from Twitter
gt1 = time()
response = api.get(endpoint="2/tweets/search/all",
params=create_params(feature, next_token,
start_time, end_time),
couchdb=couchdb,
redis=redis)
gt2 = time()
print("%.2fs to get %s tweets from %s" %
(gt2 - gt1, len(response["data"]) if "data" in response else 0,
feature["name"]))
update_location_info(feature, start_time, end_time, backfill, couchdb)
if "data" not in response or len(response["data"]) == 0:
break
tweets = format_response(response, feature)
# Add category prediction
pt1 = time()
for tweet in tweets:
predictor = Predictor(tweet["text"],
model,
tokenizer,
threshold=0.55)
prediction = predictor.get_tweet_prediction()
tweet["category"] = prediction
pt2 = time()
print("%.2fs to predict" % (pt2 - pt1))
# Save to CouchDB
t1 = time()
couchdb.connect()
couchdb["twitter"].bulk_docs(tweets)
couchdb.disconnect()
t2 = time()
print("%.2fs to save" % (t2 - t1))
def main(args):
for input_dir in args.input_dir:
if not os.path.exists(input_dir):
raise FileNotFoundError(f'{input_dir} does not exist.')
predicts_list = []
for i, input_dir in enumerate(args.input_dir):
print()
logging.info(f'Loading config from {input_dir}')
config_path = os.path.join(input_dir, 'config.json')
with open(config_path) as f:
config = json.load(f)
# load test data and word embedding once
if i == 0:
logging.info('Loading test data...')
with open(config['test_path'], 'rb') as f:
test = pickle.load(f)
assert test.shuffle is False
logging.info('Loading preproprecessed word embedding...')
with open(config['embedding_path'], 'rb') as f:
embedding = pickle.load(f)
test.context_padded_len = config['test_context_padded_len']
test.option_padded_len = config['test_option_padded_len']
config['model_parameters']['embedding'] = embedding
predictor = Predictor(training=False,
metrics=[],
device=args.device,
**config['model_parameters'])
model_path = os.path.join(input_dir, 'model.{}.pkl'.format(args.epoch))
logging.info('Loading model from {}'.format(input_dir))
predictor.load(model_path)
logging.info('Start predicting.')
start = time.time()
predicts = predictor.predict_dataset(test, test.collate_fn)
end = time.time()
total = end - start
hrs, mins, secs = int(total // 3600), int(
(total % 3600) // 60), int(total % 60)
logging.info('End predicting.')
logging.info(f'Total time: {hrs}hrs {mins}mins {secs}secs.')
predicts_list.append(predicts)
del predictor
predicts_list = torch.stack(predicts_list, dim=-1)
predicts = torch.mean(predicts_list, dim=-1) # the ensemble predictions
output_path, filename = os.path.split(
args.predict_csv_dir[:-1]
) if args.predict_csv_dir[-1] == '/' else os.path.split(
args.predict_csv_dir)
write_predict_csv(predicts, test, output_path, filename)
def main():
print("-- Welcome to movie-recommend! --")
# for output readability
np.set_printoptions(formatter={"float_kind": "{:25f}".format})
# baseline predictor by default
mode = BASELINE
# read command-line argument, if provided
if len(sys.argv) > 1:
if sys.argv[1] == IMPROVED or sys.argv[1] == BASELINE:
mode = sys.argv[1]
print(" You chose: {} predictor!".format(mode))
else:
print(
" {} is not a valid argument. Default: {} predictor!".format(
sys.argv[1], mode
)
)
else:
print(
" You did not provide any arguments. Default: {} predictor!".format(mode)
)
# read and parse text files
parser = Parser(mode)
print(" Parser initialized:")
print(
" {} test points and {} training points".format(
len(parser.test_set), np.count_nonzero(parser.training_matrix)
)
)
# initialize predictor and calculate rmse
predictor = Predictor(mode, parser.training_matrix, parser.test_set)
print(" rmse on test data (baseline): {}".format(predictor.rmse_test))
if predictor.mode == BASELINE:
print(
" rmse on training data (baseline): {}".format(predictor.rmse_training)
)
else:
print(
" rmse on test data (improved): {}".format(predictor.rmse_test_improved)
)
# execute histogram plotting and get error distribution
error_dist = (
predictor.calculate_absolute_errors(parser.test_set, predictor.improved_matrix)
if predictor.mode == IMPROVED
else predictor.calculate_absolute_errors(
parser.test_set, predictor.baseline_matrix
)
)
print(" Histogram saved to file. Error distribution: {}".format(error_dist))
def learn_parameters(hpoGraph, uni2hpoDict, dataset):
out.writeDebug('Start training the predictor.')
from predictor import Predictor
neuralNet = Predictor(None)
# in crosstraining, the test set is the crossTrain and the crossTrain set is the (here ignored) test set
crossTrainSet = {'train': dataset['train'], 'crossTrain': dataset['test'], 'test': dataset['crossTrain']}
trainingNodes = train_result_set(hpoGraph, uni2hpoDict, crossTrainSet)
out.writeDebug('Collected all the nodes for training')
if shortcut:
neuralNet.trainprediction(trainingNodes, maxEpochs = 10)
else:
neuralNet.trainprediction(trainingNodes)
return neuralNet
class Player(object):
def __init__(self, filename):
self.gamestate = go.PyGoGame("".encode('ascii'))
self.predictor = Predictor(filename)
def clear(self):
self.gamestate = go.PyGoGame("".encode('ascii'))
def size(self):
return 19
def move(self, color, move):
color = go.BLACK if (color[0].lower() == 'b') else go.WHITE
if move.lower() == 'pass':
self.gamestate.make_move(color, -1, -1)
else:
row = int(move[1:])
col = 1 + board_letters.index(move[0].lower())
self.gamestate.make_move(color, row, col)
def genmove(self, color):
self.gamestate.rewind()
# play game to end
for idx in range(self.gamestate.gamelen()):
self.gamestate.next_move()
self.gamestate.set_to_play(go.BLACK if (color[0].lower() == 'b') else go.WHITE)
features = np.zeros((NUM_FEATURES, 19, 19), dtype=np.uint8)
offset = range(NUM_FEATURES + 1).__iter__()
def next_feature(val):
features[next(offset), ...] = val
get_current_features(self.gamestate, next_feature)
predictions = self.predictor.predict(features).ravel()
predictions_board = predictions[:361].reshape((19, 19))
# mask to legal moves - will modify predictions as well
legal = features[-1, ...]
# print(legal.shape, features.shape, predictions_board)
predictions_board[legal == 0] = 0
# apply temperature
probs = np.exp(predictions - predictions.max()) # pin most likely prediction to 1.0
probs **= 0.9 # slightly warm temp ==> more random behavior
probs /= probs.sum()
# choose a random legal move
move = np.random.choice(362, p=probs)
if move == 361:
return 'pass'
else:
row = move // 19 + 1
col = move % 19 + 1
return "{:d} {:d}".format(col, row)
def simulate_similarity_threshold():
sim_thresholds = np.arange(-0.0, 0.30, 0.02)
p.PREDICT_BASE_SIM_ONLY = True
results = []
for threshold in sim_thresholds:
p.SIMILARITY_THRESHOLD = threshold
predictor = Predictor()
result = predictor.predict_all()
results.append(result)
plt.plot(sim_thresholds, results)
plt.title("Similarity Threshold Tuning")
plt.xlabel("Similarity Threshold")
plt.ylabel("MSE for Baseline + Similarity")
plt.savefig(p.SIMILARITY_TUNING_FILE_OUT)
def __init__(self, **kwargs):
total_start = datetime.now()
test = kwargs['filename'] or input('\nEnter test name: ')
test_rating = parse(os.path.join(TEST_DIR, test + '.test'))
base_rating = parse(os.path.join(TEST_DIR, test + '.base'))
graph = Graph(base_rating=base_rating)
while True:
try:
cluster_number = kwargs['cluster_num'] or int(input('\nEnter number of clusters: '))
component = BoruvkaMST(graph, cluster_number)
break
except (ClusterNumberException, ValueError):
print('Number of clusters must be > 0')
continue
predictor = Predictor(component, base_rating)
from copy import deepcopy
predicted = deepcopy(test_rating)
unpredictables = 0
start = datetime.now()
with open(os.path.join(TEST_DIR, test + '.res'), newline='', mode='w') as csv_file:
data_writer = csv.writer(csv_file, delimiter=' ', quotechar='|')
for user in test_rating.keys():
for movie in test_rating[user].keys():
predicted[user][movie] = None
predicted[user][movie] = predictor.predict(user, movie)
if not predicted[user][movie]:
unpredictables += 1
data_writer.writerow([user, movie, predicted[user][movie]])
self.predicting_time = datetime.now() - start
print('Time for predicting:', self.predicting_time, end='\n\n')
self.total_time = datetime.now() - total_start
print('Total time:', self.total_time)
print('Number of unpredictable users:', unpredictables)
self.evaluator = Evaluator(test_rating, predicted)
def predict(load=False, keep_going=False):
# predict sentence
if load:
model.load_state_dict(torch.load(args.save))
print("model restored")
predictor = Predictor(model, abstracts.vectorizer)
count = 1
while True:
seq_str = input("Type in a source sequence:\n")
seq = seq_str.strip().split(' ')
num_exams = int(input("Type the number of drafts:\n"))
print("\nresult:")
outputs = predictor.predict(seq, num_exams)
for i in range(num_exams):
print(i)
print(outputs[i])
print('-' * 120)
count += 1
if not keep_going and count > config.predict_right_after:
break
def main():
start_time = time()
predictor = Predictor(company='INDEX', dataset='Bloomberg', classifier='NN', feature_type='BoW')
predictor.run()
# companies = ['GOOG', 'BA', 'WMT', 'INDEX']
# # datasets = ['Bloomberg', 'Reuters']
# classifiers = ['SVM', 'NN']
# feature_types = ['BoW', 'Event']
# for company in companies:
# for classifier in classifiers:
# for feature_type in feature_types:
# predictor = Predictor(company=company, dataset='Bloomberg', classifier=classifier, feature_type=feature_type)
# print '##############################'
# print 'Company:', company
# print 'Classifier:', classifier
# print 'Feature type:', feature_type
# predictor.run()
print '----------' + str(round(time() - start_time, 2)) + ' seconds.---------------'
class MinimaxPlayer(Player):
"""Minimax player attempts to play with minimax rules."""
THRESHOLD = 0.8
def __init__(self, name, depth = 2, breadth = 3):
Player.__init__(self, name)
self.depth = depth
self.breadth = breadth
self.predictor = Predictor(self.THRESHOLD)
def round_start(self, game_state):
self.predictor.reset(game_state)
def choose_card(self, game_state):
self.predictor.refresh(game_state, self)
players = self.predictor.players
turn_index = players.index(self)
def utility(pred, trick):
total_tricks = len(pred.deck.cards) / whist.NUM_PLAYERS
played = [card for card, player in pred.plays.items() if player == self]
tricks_remaining = total_tricks - len(played)
not_played = [card for card in self.cards if card not in played]
win_thresh = (tricks_remaining * len(ranks)) / total_tricks
score = 0
for card in not_played:
if ranks.index(card.rank) >= win_thresh:
score += 1
else:
score -= 1
return score
def vopt(pred, trick, depth):
if depth == 0:
return utility(pred, trick)
score = 0
turn = (players.index(trick.play_order[-1]) + 1) % whist.NUM_PLAYERS
# first we calculate the score if possible
if not trick.left_to_play: # all players have played
if game_state.are_partners(self.name, trick.winning_player().name):
score = 1
else:
score = -1
turn = players.index(trick.winning_player())
if not players[turn].cards: # game complete
return score
trick = Trick(players, game_state.trump)
depth = depth - 1
cards = util.get_legal_cards(pred.predict(players[turn]), trick.suit_led)
cards = cards if len(cards) < self.breadth else random.sample(cards, self.breadth)
plays = []
# next we attempt to play cards
if not cards: # this branch cannot be satisified, prune it
return 0
for card in cards:
trick.play_card(players[turn], card)
pred.try_play(players[turn], card)
plays.append(score + vopt(pred, trick, depth))
pred.revert_play(players[turn], card)
trick.revert_play(players[turn], card)
# finally we return the highest score
if game_state.are_partners(self.name, players[turn].name): # max
return max(plays)
else: # min
return min(plays)
# make best play
legal_cards = util.get_legal_cards(self.cards, game_state.trick.suit_led)
trick = game_state.trick
choices = []
max_score = float("-inf")
pred = self.predictor
for card in legal_cards:
trick.play_card(self, card)
pred.try_play(self, card)
score = vopt(pred, trick, self.depth)
pred.revert_play(self, card)
trick.revert_play(self, card)
print "Predict %s: %d" % (card, score)
choices.append((score, card))
if score > max_score:
max_score = score
choices = [card for score, card in choices if score == max_score]
max_card = random.choice(choices)
pred.learn(self, max_card)
return max_card
def observe_play(self, game_state, player, card):
self.predictor.refresh(game_state, self)
self.predictor.learn(player, card)
def __init__(self, dst_point, vertex_box): Predictor.__init__(self, dst_point, vertex_box) self.vertex_box = self.neighbor_lst matrix_str = self.__genmatrix() self.A = np.matrix(matrix_str)
def initialWeightsComputed(self, initialWeights):
#create Predictor Object
#cherrypy.log("initial weights computed", context='DEBUG')
self.predictor = Predictor(self.traffic,initialWeights)
# -*- coding: utf-8 -*-
#import sys
#reload(sys)
#sys.setdefaultencoding('utf-8')
from predictor import Predictor
predict=Predictor()
#target for below:{"death_penalty": false, "imprisonment": 8, "life_imprisonment": false}, "punish_of_money": 0, "accusation": ["盗窃"], "relevant_articles": [264]},
content1=u"经审理查明：一、2016年9月20日，被告人胡某某伙同罗某某（13岁）将被害人龙某甲停放在桂阳县欧阳海乡周塘村7组龙某某家门前水泥坪的一辆雅奇牌男式两轮摩托车盗走，并将该摩托车以300元的价格卖掉，被告人胡某某分得100元。案发后，公安机关已将追回的被盗摩托车发还给被害人龙某甲。经鉴定，被盗摩托车的价值为1330元。二、2016年9月21日，被告人胡某某伙同罗某某将被害人刘某甲停放在桂阳县仁义镇烟草站仓库门口空坪处的一辆豪江牌男式两轮摩托车盗走，并将该摩托车以300元的价格卖掉，被告人胡某某分得100元。案发后，公安机关已将追回的被盗摩托车发还给被害人刘某甲。经鉴定，被盗摩托车的价值为1580元。三、2016年9月23日，被告人胡某某伙同罗某某将被害人李某甲停放在桂阳县敖泉卫生院内空坪处的一辆豪爵牌男式两轮摩托车盗走，后两人与张某甲一起将该摩托车拖走后以800元的价格卖掉，被告人胡某某分得300元。案发后，公安机关已将追回的被盗摩托车发还给被害人李某甲。经鉴定，被盗摩托车的价值为2800元。上述事实，被告人胡某某在庭审时也无异议，且有受案登记表、立案决定书，抓获经过，健康检查登记表，被盗摩托车的相关材料，扣押物品决定书，扣押物品清单，发还物品清单，价格鉴定意见，行政处罚决定书，责令监管通知书，证人史某某、王某某、罗某某、张某甲的证言，被害人刘某甲、李某甲、龙某甲的陈述，被告人胡某某的供述与辩解，现场勘查笔录及照片，现场指认笔录及照片，辨认笔录，监控视频及视频截图，户籍资料等证据证实，足以认定。"
#targe for below:{"death_penalty": false, "imprisonment": 8, "life_imprisonment": false}, "punish_of_money": 4000, "accusation": ["盗窃"], "relevant_articles": [264]},
content2=u"上海市静安区人民检察院指控被告人童某与“昆昆”(另案处理)共同于2016年3月1日16时许，在本市静安区江宁路XXX号XXX楼走道内，趁人不备，窃得被害人薛某停放在此的一辆价值人民币7,298元的白色标志牌QP100T-12C型摩托车，后逃逸销赃。被告人童某于2016年4月7日主动至上海市公安局静安分局投案自首。"
#target for below:{"death_penalty": false, "imprisonment": 10, "life_imprisonment": false}, "punish_of_money": 2000, "accusation": ["盗窃"], "relevant_articles": [264]}
content3=u"公诉机关指控：2013年11月7日11时许，被告人钟某某在佛山市高明区明城镇某某金属有限公司工作时，趁其他员工去吃午饭时将公司的19条黄铜锭从公司厂房墙角的洞口丢到外面，并于2013年11月8日2时许独自驾驶一辆女装摩托车到公司厂房墙外将其中的15条黄铜锭盗走并销赃给他人。2013年11月8日11时许，被告人钟某某以同样的方式将11条黄铜锭丢到公司厂房外面，并于2013年11月10日2时许独自驾驶一辆女装摩托车准备把剩余的15条黄铜锭盗走，行至公司厂房墙外时被伏击的公安人员抓获。公安人员扣押了上述15条黄铜锭，并已发还给被害人。经价格鉴定，被盗的30条黄铜锭共价值人民币13884元。公诉机关认为被告人钟某某无视国家法律，以非法占有为目的，盗窃他人财物，价值人民币13884元，数额较大，其行为已触犯了《中华人民共和国刑法》××之规定，应以××追究其刑事责任。在法庭上，公诉机关还详细阐述了关于指控被告人盗窃数额为13884元的相关依据和计算方法。公诉机关建议对被告人钟某某判处六个月至一年六个月××，并处罚金。提请本院依法判处。"
#{"death_penalty": false, "imprisonment": 3, "life_imprisonment": false}, "punish_of_money": 1000, "accusation": ["盗窃"], "relevant_articles": [264]}
content4=u"昆山市人民检察院指控并经本院审理查明，2014年4月17日凌晨，被告人熊某伙同冯某、张某、杨某（另案处理）至昆山市锦溪镇阮家浜村X组X号XX超市，由张某用手拉弯防盗窗栏杆，被告人熊某、张某、冯某在外望风，杨某从窗户爬进店内，窃得人民币1300余元；三星牌GT-I8552型手机1部（价值人民币904元）；南京（特醇）香烟5包（价值人民币55元）；南京（佳品）香烟5包（价值人民币75元）；黄鹤楼（硬雅香金）香烟2包（价值人民币40元）；利某（新版）香烟1包（价值人民币13元）；苏某（五星红杉树）香烟1包（价值人民币20元）。被告人熊某归案后如实供述了犯罪事实。另查明，案发后公安机关已将扣押的涉案香烟十四包、手机一部、现金人民币860元已发还被害人阮某。在本院审理期间，被告人熊某退缴人民币440元及预缴罚金保证金人民币1000元，暂扣于本院。上述事实，有公诉机关提交，并经庭审质证、认证的证人张某、冯某、杨某等人的证言，被害人阮某的陈述，昆山市价格认证中心出具的价格鉴证结论书，搜查笔录，现场勘验笔录，监控录像，扣押、发还物品清单，抓获经过，身份信息等证据予以证实，被告人熊某亦供认不讳，且与上述证据证实的事实相吻合，证据均具有证明效力，本院予以确认。"
#{"death_penalty": false, "imprisonment": 11, "life_imprisonment": false}, "punish_of_money": 0, "accusation": ["故意伤害"], "relevant_articles": [234]}
content5="北京市密云县人民检察院指控，被告人邓×5于2013年11月17日15时50分许，在自己家中，因琐事与尹×2发生口角，后被告人邓×5持菜刀将尹×2头部和脖子砍伤。经鉴定，尹×2身体所受损伤程度属轻伤。公诉机关向本院提供了相关证据，认为被告人邓×5有自首情节，提请依照《中华人民共和国刑法》××的规定，对被告人邓×5处罚。"
#"imprisonment": 80, "life_imprisonment": false}, "punish_of_money": 4000, "accusation": ["强奸", "盗窃"], "relevant_articles": [264]}
content6="河北省武安市人民检察院指控，2008年8月9日凌晨，被告人王丁某伙同张2某（在逃）合谋后，窜至武安市冶陶镇马村，趁无人之机，将张某停放在家门前的一辆邯郸牌150型拖拉机盗走，后二人将拖拉机卖到阳邑镇刘某（已判）的废品收购站，得款几千元，所得赃款平分后挥霍，经武安市价格认证中心鉴定，被盗拖拉机价值6440元。破案后，赃物被追回，退还失主。"
#??
#content7="经审理查明，1994年12月1日13时许，被告人张某与同村村民李某（已判刑）、陈xx及被害人祝xx（右上肢残疾）等人在范县陆集乡东街饭店喝酒，期间发生矛盾，祝xx将酒桌掀翻。被人劝开后，被告人张某及李某、陈xx离开饭店，祝xx从饭店追至范县陆集乡工商所北边坑洼处，祝xx挥舞菜刀将李某的右手臂砍伤，李某遂将祝xx踹倒，捡起掉落在地上的菜刀朝祝xx头部连砍数刀，并用大衣将祝xx头部盖住，祝xx将大衣从脸上拉下，李某又拉起大衣重新盖住祝xx头部，祝xx倒在地上。被告人张某见状便从附近村民家拿来铁锨掩埋血迹，并铲土压在祝xx的大衣边缘。随后张某伙同他人又到另一饭店喝酒，当听说有人报案时逃跑。被害人祝xx经抢救无效于1994年12月3日上午9时许死亡。经检验，祝xx死于失血性休克合并颅脑损伤。经本院查证核实，祝xx的亲属对张某表示谅解，并请求对张某从轻处罚。上述事实，被告人在庭审过程中也无异议，且有证人李某、陈xx、王xx、孙xx、李xx、夏xx、张xx、马xx等人的证言，现场勘查笔录、现场方位示意图、现场照片，范县公安局尸体检验笔录，提取笔录（铁锹），对张某的辨认笔录，对李某的刑事附带民事判决书，抓获经过等证据证实，足以认定。"
content_list=[content1,content2,content3,content4,content5,content6]
result=predict.predict(content_list)
print("length of result:",str(len(result)));
for i,e in enumerate(result):
print(i,e)
rankings = [(total/simSums[item] + self.means[person],item) for item, total in totals.items()] #todo pass 100 at most or so
else:
rankings = [(total/simSums[item],item) for item, total in totals.items()] #todo pass 100 at most or so
# Return the sorted list
rankings.sort()
rankings.reverse()
return rankings
if __name__=="__main__":
'''
sys.argv[1] => training data
sys.argv[2] => data separator
'''
a = Predictor(sys.argv[1], sys.argv[2])
users, items = a.store_data_relations() #~100MB
#ratings, means = a.normalize_ratings(users)
recommender = UserBasedPredictor(users) #first, without normalizing
#recommender = UserBasedPredictor(ratings, means)
#todo: checar quais os itens mais recomendados e se eles sao populares ou oq
#todo: estudar o nivel de personalizacao desse babado
b = Ranker(5)
statistically_better = 0.0
for u in users.keys():
#print u, b.topRatings(recommender.getRecommendations(u)[:30])
a = b.maximizeKGreatItems(1, recommender.getRecommendations(u)[:60], items)
def __init__(self, name, depth = 2, breadth = 3):
Player.__init__(self, name)
self.depth = depth
self.breadth = breadth
self.predictor = Predictor(self.THRESHOLD)
class PredictionEngine(plugins.Monitor):
exposed = True
def __init__(self, bus, sleeping_time, traffic):
plugins.Monitor.__init__(self, bus, self.predictionEngine, sleeping_time)
self.bus.subscribe(progaconstants.UPDATED_TRAFFIC_CHANNEL_NAME,self.trafficUpdated)
self.bus.subscribe(progaconstants.INITIAL_WEIGHTS_COMPUTED_CHANNEL_NAME,self.initialWeightsComputed)
self.bus.subscribe(progaconstants.SIMULATION_STOPPED_CHANNEL_NAME,self.simulationFinished)
self.bus.subscribe(progaconstants.SIMULATION_FINISHED_CHANNEL_NAME,self.simulationFinished)
self.bus.subscribe(progaconstants.SIMULATION_STARTED_CHANNEL_NAME,self.simulationStarted)
self.monitor_active = False;
self.sleeping_time = sleeping_time
self.traffic = traffic
self.predictor = None
"""
prediction engine viene richiamata ogni sleeping_time secondi (vedi file progaconstant alla voce PREDICTION_SLEEP_SECONDS)
per il momento e' impostata a 2. Possiamo usare questo hook per svolgere attivita' di contorno relative alla predizione.
In teoria tutte quelle diverse dall'aggiornamento pesi che viene fatto dal Predictor su stimolo della funzione trafficUpdated
"""
def predictionEngine(self):
pass
#cherrypy.log("prediction engine running", context='DEBUG')
#pass
#cherrypy.log("asking for traffic", context='DEBUG')
#currentState = self.traffic.getTraffic()
def initialWeightsComputed(self, initialWeights):
#create Predictor Object
#cherrypy.log("initial weights computed", context='DEBUG')
self.predictor = Predictor(self.traffic,initialWeights)
def trafficUpdated(self, elapsedSeconds):
#cherrypy.log("Updating traffic", context='DEBUG')
v = self.traffic.getTraffic()
#for t in v.values():
#cherrypy.log("id:"+str(t['flight_id']) + " x:" + str(t['x']) + " y:"+str(t['y']) + " z:"+str(t['z']) + " lat:"+str(t['lat'])+ " lon:"+str(t['lon']), context='TRAFFIC')
self.predictor.trafficUpdated(elapsedSeconds)
def simulationFinished(self):
#cherrypy.log("I AM PREDICTION ENGINE AND I KNOW SIMULATION IT'S FINISHED", context='DEBUG')
self.unsubscribe()
self.stop()
def simulationStarted(self, t0):
self.subscribe()
self.start()
cherrypy.log("SIMULATION ENGINE STARTING", context='AUTO')
self.predictor.simulationStarted(t0)
def toBool(self, s):
return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'uh-huh']
'''
in questa funzione mettiamo il risultato del sensing di potenziali conflitti
il monitoraggio non verra fatto automaticamente dal server ma richiesto continuamente
dal client per tutta la durata di attivazione della funzione
in altre parole ci pensa il client a tenersi informato sullo stato dei conflitti potenziali
'''
def checkConflicts(self, ownPos, ownVel, flight_IDs, deltaT, nsteps, ownIntent=None):
"""
Input:
- ownPos, current position of monitored aircraft
- ownVel, current velocity of monitored aircraft
- ownIntent, flight intent of monitored aircraft
- ownID, flight ID of monitored aircraft
- flight_IDs, flight intent of surrounding traffic
- deltaT, prediction unit time interval
- nsteps, number of predictions
"""
potentialConflicts = {}
timeHorizons = [i*deltaT for i in range(1, nsteps+1)]
fp = futurePositions(ownPos, ownVel, ownIntent, timeHorizons)
prediction = self.predictor.predictionRequested(flight_IDs, deltaT, nsteps, True)
ztp = zip(timeHorizons, fp)
for aID, foo in prediction.items():
predDict = foo[0]
for t, p in ztp:
#cherrypy.log("Ownship will be at %s in %d seconds" % (p, t),context="MONITOR")
#cherrypy.log("%s will be at %s in %d seconds" % (aID, sum(predDict[t])/len(predDict[t]), t),context="MONITOR")
for q in predDict[t]:
if euclideanDistance(p,q) < ALERT_DISTANCE:
potentialConflicts.setdefault(t, []).append(aID)
break
#pdb.set_trace()
return potentialConflicts
@cherrypy.tools.accept(media='text/plain')
def GET(self, flight_id, deltaT, nsteps, raw, coords_type=progaconstants.COORDS_TYPE_GEO):
if flight_id == progaconstants.MONITOR_ME_COMMAND:
ownship_state = self.traffic.getMyState()
v = np.array([ownship_state['vx'],ownship_state['vy'],ownship_state['vz']])*FOOT2MT
p = Point3D(ownship_state['lon'], ownship_state['lat'], ownship_state['h']*FOOT2MT).getNumpyVector()
#pdb.set_trace()
fids = self.traffic.getActiveFlightIDs()
ownship_intent = self.traffic.getOwnshipIntent()
intruders = self.checkConflicts(p,v,fids,60,10,ownship_intent)
#pdb.set_trace()
return json.dumps(intruders)
else:
flight_IDs = [flight_id]
deltaT = int(deltaT)
nsteps = int(nsteps)
rawPrediction = self.toBool(raw)
prediction_matrix = self.predictor.predictionRequested(flight_IDs, deltaT, nsteps,rawPrediction)
#pdb.set_trace()
#RAW PREDICTION WAS REQUESTED, WE PROVIDE PARTICLES POSITIONS
if rawPrediction:
for flight in prediction_matrix:
for times in prediction_matrix[flight][0]:
#prediction_matrix[flight][0] e' l'elemento che contiene i valori di predizione
#mentre [1] contiene le leg utilizzate per predire quel volo
for i in range(0,len(prediction_matrix[flight][0][times])):
#for tris in range(0,len(prediction_matrix[flight][0][times][i])):
#pdb.set_trace()
if (coords_type == progaconstants.COORDS_TYPE_GEO):
p3d = Point3D()
#pdb.set_trace()
vect = p3d.lonLatAltFromXYZ(prediction_matrix[flight][0][times][i][0], prediction_matrix[flight][0][times][i][1], prediction_matrix[flight][0][times][i][2])
prediction_matrix[flight][0][times][i] = vect
prediction_matrix[flight][0][times] = prediction_matrix[flight][0][times].tolist()
jmat = json.dumps(prediction_matrix)
cherrypy.log("%s" % (jmat), context="PREDICTION")
return jmat
#NORMAL PREDICTION WAS REQUESTED, WE PROVIDE BINS OF PROBABILITY
else:
#pdb.set_trace()
for flight in prediction_matrix:
for dt in prediction_matrix[flight]:
prediction_matrix[flight][dt][0] = prediction_matrix[flight][dt][0].tolist()
for i in range(0,len(prediction_matrix[flight][dt][1])):
prediction_matrix[flight][dt][1][i] = prediction_matrix[flight][dt][1][i].tolist()
jmat = json.dumps(prediction_matrix)
cherrypy.log("%s" % (jmat), context="PREDICTION")
#scrivi qui codice di test
#cherrypy.log("%s" % prediction_matrix[flight_IDs[0]][deltaT][0], context="TEST")
return jmat
def POST(self,command=''):
if command == 'start':
cherrypy.log("starting prediction engine", context='AUTO')
self.subscribe()
self.start()
if command == 'stop':
self.unsubscribe()
self.stop()
def PUT(self,another_string):
pass
def DELETE(self):
pass
for (item, rating) in user_ratings.items():
for item2 in selected_items:
scores.setdefault(item2,0)
if item2 in self.similar_items[item]:
scores[item2] += self.similar_items[item][item2] * rating
rankings = [(score,item) for item,score in scores.items()]
rankings.sort(); rankings.reverse()
return rankings
if __name__=="__main__":
'''
sys.argv[1] => training data
sys.argv[2] => test data
sys.argv[3] => data separator
'''
training = Predictor(sys.argv[1], sys.argv[3])
training_users, training_items = training.store_data_relations() #~100MB
recommender = NNCossNgbrPredictor(training_items, training_users)
N = 10
ranker = Ranker(N)
testing = Predictor(sys.argv[2], sys.argv[3])
test_users, test_items = testing.store_data_relations()
ev = Evaluator(test_users, N)
#TODO clean this interface!
item_ids = list(set(training_items.keys() + test_items.keys())) #all unique items in the dataset
hits = 0
div_metric1 = []
div_metric2 = []
print("start training...")
model = train_epoches(abstracts, model, config.epochs, teacher_forcing_ratio=1)
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early')
torch.save(model.state_dict(), args.save)
print("model saved")
predict()
elif args.mode == 1:
predict(load=True, keep_going=True)
elif args.mode == 2:
num_exams = 3
# predict file
model.load_state_dict(torch.load(args.save))
print("model restored")
predictor = Predictor(model, abstracts.vectorizer)
data_path = cwd + config.relative_dev_path
abstracts = headline2abstractdataset(data_path, abstracts.vectorizer, args.cuda, max_len=1000)
print("number of test examples: %d" % len(abstracts))
f_out_name = cwd + config.relative_gen_path
outputs = []
title = []
for j in range(num_exams):
outputs.append([])
i = 0
print("Start generating:")
train_loader = DataLoader(abstracts, config.batch_size)
for batch_idx, (source, target, input_lengths) in enumerate(train_loader):
output_seq = predictor.predict_batch(source, input_lengths.tolist(), num_exams)
for seq in output_seq:
title.append(seq[0])
def __init__(self, flm_spec):
Predictor.__init__(self, order=flm_spec.order())
self.flm_spec = flm_spec
lambda v, nv: any(word in v for word in ["hair", "holidays"]))
if c>0:
print "Imputed {0} values of '{1}' with '{2}'".format(c, feature,
new_value)
print "Preparing dataset and targets before entering algorithm..."
t0 = time()
dataset.split()
dataset.transform_dataset()
# Split in train and test arrays
X_train, X_test, y_train, y_test = train_test_split(
dataset.get_dataset(), dataset.get_targets(), test_size=.30)
print "DONE. Time preparing dataset: {:.2f}s".format(time()-t0)
print "Modeling predictor... "
predictor = Predictor(n_estimators=100)
t0 = time()
predictor.fit_algorithm(X_train, y_train)
prediction = predictor.predict_outputs(X_train)
t1 = time() - t0
print "DONE. Time modeling predictor: {0:.2f}s.".format(t1)
mse = predictor.predictor_metrics(y_train, prediction)
print "MSE on train set: {0:.3f}".format(mse)
print "Testing predictor... "
t0 = time()
score = predictor.score_predictor(X_test, y_test)
prediction = predictor.predict_outputs(X_test)
mse = predictor.predictor_metrics(y_test, prediction)
t1 = time() - t0
print "DONE. Time testing predictor: {0:.2f}s.".format(t1)
import sys
from data.stock import Stock
from predictor import Predictor
from score import auc
from preprocess import Preprocess
submission_number = sys.argv[1]
print submission_number
def submission(ids, preds):
name = "submissions/submission{0}.csv".format(submission_number)
f = open(name, 'w')
if len(ids) != len(preds):
raise Exception("The number of IDs and the number of predictions are different")
string = 'id,prediction\n'
for index,i in enumerate(ids):
string += str(i) + ',' + str(preds[index]) + "\n"
f.write(str(string))
f.close()
data, targets, _ = Stock.train()
holdout_data, ids = Stock.test()
assert len(data) == len(targets)
Predictor.train(data, targets)
preds = Predictor.multi_predict(holdout_data)
print preds[0:50]
submission(ids, preds)
def __init__(self, filename):
self.gamestate = go.PyGoGame("".encode('ascii'))
self.predictor = Predictor(filename)
from data.stock import Stock from predictor import Predictor from score import auc from preprocess import Preprocess data, targets, cv_targets = Stock.train() #data = Preprocess.scale(data) #cv_targets = Preprocess.scale(cv_targets) #data = Preprocess.polynomial(data, 5) half = len(data)/2 tr_data, holdout_data = data[:half], data[half:] train_targets, holdout_targets = targets[:half], targets[half:] Predictor.train(tr_data, train_targets) preds = Predictor.multi_predict(holdout_data) print 'AUC ', auc(preds, cv_targets) print Predictor.multi_predict([[0.]]) print Predictor.multi_predict([[0.1]]) print Predictor.multi_predict([[-0.1]]) print Predictor.multi_predict([[0.01]]) print Predictor.multi_predict([[-0.01]])
def prediction(config, algorithm, feature, logger, good_labels, manager):
print "Start predictions and checks..."
from predictor import Predictor
checker = Predictor()
checker.set_algorithm(algorithm)
checker.set_feature(feature)
checker.set_logger(logger)
checker.set_good_labels(good_labels)
checker.set_db_file(config.get_db_file())
from result import ResultPrediction
checker.set_resultmanager(ResultPrediction(logger, good_labels, manager))
if config.is_check():
print "Running Checks..."
print "Used for checking the accuracy of the IDS"
checker.runner(config.get_check_sets(), config, good_labels)
print "Checks done"
print "End predictions and checks.\n"
self.data_train = data_train
self.data_test = data_test
self.k_neighbors = k_neighbors
def train(self):
pass
samples = [[0., 0., 0.], [0., .5, 0.], [1., 1., .5]]
y = [2,1,3]
from sklearn.neighbors import NearestNeighbors
neighbors = 3
neigh = NearestNeighbors(n_neighbors=neighbors)
neigh.fit(samples)
NearestNeighbors(algorithm='auto', leaf_size=1)
# algorithm : {‘auto’, ‘ball_tree’, ‘kd_tree’, ‘brute’}, optional
#
# Algorithm used to compute the nearest neighbors:
#
# ‘ball_tree’ will use BallTree
# ‘kd_tree’ will use KDTree
# ‘brute’ will use a brute-force search.
# ‘auto’ will attempt to decide the most appropriate algorithm based on the values passed to fit method.
#
for i in range(0,neighbors):
print neigh.kneighbors([1., 1., 1.])[0][0][i]
print neigh.kneighbors([1., 1., 1.])[1][0][i]
print Predictor.predict(3, neigh.kneighbors([1., 1., 1.]), y)
