def testIrisNoisy(trainDataFile, testDataFile, attrDataFile):
irisTree = DecisionTree(trainDataFile, testDataFile, attrDataFile)
irisTree.loadData()
irisTree.preprocessData()
irisTree.splitDataSet(irisTree.getTrainData())
trainData = irisTree.getTrainData()
testData = irisTree.getTestData()
attributes = irisTree.getAttributes()
attrValues = irisTree.getAttrValues()
validationData = irisTree.getValidationData()
print(validationData)
for rate in range(0, 21, 2):
noisyData = addNoise(trainData, rate, irisTree.getClasses())
irisTree.generateTree(noisyData)
tree = irisTree.getTree()
irisRulePrinter = RuleSetPrinter(tree, attrValues)
print("\nNoise Rate (%): " + str(rate))
print("--------Rule Set--------")
irisRulePrinter.printRuleSet()
rules = irisRulePrinter.getRules()
testPred = Predictor(testData, validationData, attributes, attrValues,
tree, rules)
# testAcc = testPred.calculateRuleAccuracy(validationData, rules)
testAcc = testPred.calculateRuleAccuracy(testData, rules)
print("Test Accuracy (%): " + str(testAcc))
print("----Post-pruning Rules----")
pruneRules = testPred.postPruning()
print(os.linesep.join(pruneRules))
testPred = Predictor(testData, validationData, attributes, attrValues,
tree, pruneRules)
testAcc = testPred.calculateRuleAccuracy(testData, pruneRules)
print("Test Accuracy (%): " + str(testAcc))
def main(filename):
#Loading data
dataset = DataLoader(filename)
explorer = DataExplorer(dataset)
predictor = Predictor(dataset)
#Exploring data
explorer.visualise_distributions(without_outliers=False)
explorer.visualise_distributions(without_outliers=True,
identify_abnormal=True)
explorer.describe_variable(652)
explorer.histogram(652, without_outliers=True)
explorer.boxplot(652, without_outliers=True)
#PCA
explorer.pca(plot=True)
#Correlations
explorer.best_relationship_class()
explorer.visualise_best_relationship_class()
explorer.best_relationship_pca()
explorer.visualise_best_relationship_pca()
#SVM
a = Predictor(dataset)
a.svc()
def testTennis(trainDataFile, testDataFile, attrDataFile):
tennisTree = DecisionTree(trainDataFile, testDataFile, attrDataFile)
tennisTree.loadData()
tennisTree.preprocessData()
trainData = tennisTree.getTrainData()
testData = tennisTree.getTestData()
validationData = tennisTree.getValidationData()
tennisTree.generateTree(trainData)
tree = tennisTree.getTree()
attributes = tennisTree.getAttributes()
attrValues = tennisTree.getAttrValues()
tennisTreePrinter = TreePrinter(tree, attrValues)
print("----------Tree----------")
tennisTreePrinter.printTree()
tennisRulePrinter = RuleSetPrinter(tree, attrValues)
print("--------Rule Set--------")
tennisRulePrinter.printRuleSet()
rules = tennisRulePrinter.getRules()
trainPred = Predictor(trainData, validationData, attributes, attrValues,
tree, rules)
trainAcc = trainPred.calculateAccuracy(trainData, tree)
print("Training Accuracy (%): " + str(trainAcc))
testPred = Predictor(testData, validationData, attributes, attrValues,
tree, rules)
testAcc = testPred.calculateAccuracy(testData, tree)
print("Test Accuracy (%): " + str(testAcc))
def main():
args = get_parser()
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
args.device = torch.device("cuda:0")
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
if not os.path.exists(args.save_path):
os.makedirs(args.save_path)
args.model_path = os.path.join(args.save_path, args.model_name, 'model')
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
if args.model_name == 'generator':
g = Generator(args)
if args.test:
args.model_path = os.path.join(args.save_path, 'predictor',
'model')
hs = args.hs
args.hs = 512
p = Predictor(args)
args.model_path = os.path.join(args.save_path, args.model_name,
'model')
args.hs = hs
g.meta_test(p)
else:
g.meta_train()
elif args.model_name == 'predictor':
p = Predictor(args)
p.meta_train()
else:
raise ValueError('You should select generator|predictor|train_arch')
def get_predictors_with_calibration_file(calibration_file_name):
models = get_models_with_calibration_file(calibration_file_name)
return {
"x_offsets_predictor_x_input_only": Predictor(models["x_offsets_model_x_input_only"]),
"x_offsets_predictor": Predictor(models["x_offsets_model"]),
"y_offsets_predictor_y_input_only": Predictor(models["y_offsets_model_y_input_only"]),
"y_offsets_predictor": Predictor(models["y_offsets_model"]),
"angle_offsets_predictor": Predictor(models["angle_offsets_model"])
}
def select_model(self):
model_rmse = {}
data_df = Retriever(file='/tmp/retrieve_data.log',
loglevel=logging.DEBUG).getData()
steps = [('clean_data',
Cleaner(file='/tmp/clean_data.log', loglevel=logging.DEBUG)),
('process_data',
Processor(file='/tmp/process_data.log',
loglevel=logging.DEBUG))]
p = Pipeline(steps)
data_array, labels = p.fit_transform(data_df)
# random forest
model = RandomForestRegressor(n_estimators=1000,
n_jobs=-1,
random_state=0)
predictor = Predictor(model,
5,
file='/tmp/predictor.log',
loglevel=logging.DEBUG)
score = predictor.score(data_array, labels)
model_rmse[model] = score
self.logger.info('=== Random Forest rmse : {}'.format(score))
# LGB
model = LGBMRegressor(n_estimators=1000, learning_rate=0.01)
predictor = Predictor(model,
5,
file='/tmp/predictor.log',
loglevel=logging.DEBUG)
score = predictor.score(data_array, labels)
model_rmse[model] = score
self.logger.info('=== LGB rmse : {}'.format(score))
# Baseline
_, rmse = self.predict_baseline(data_array, labels)
model_rmse[None] = rmse
self.logger.info('=== baseline rmse : {}'.format(rmse))
# LSTM
save_weights_at, test_set = self.build_save_model_LSTM(
data_array, labels, 'intrinio')
model, y, rmse = self.predict(save_weights_at, test_set)
model_rmse[model] = rmse
self.logger.info('=== baseline rmse : {}'.format(rmse))
# Select the model with min rmse
m = min(model_rmse, key=model_rmse.get)
return m, model_rmse[m]
def infer(self, args: ClassifierArgs) -> Dict:
content = args.content
assert content is not None, 'in infer mode, parameter content cannot be None! '
content = content.strip()
assert content != '' and len(
content) != 0, 'in infer mode, parameter content cannot be empty! '
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
pred_probs = predictor.predict(content)
pred_label = np.argmax(pred_probs)
pred_label = self.data_reader.get_idx_to_label(pred_label)
if pred_label == '100':
pred_label = '0'
elif pred_label == '101':
pred_label = '1'
result_in_dict = {
'content': content,
'pred_label': pred_label,
'pred_confidence': pred_probs
}
result_in_str = ', '.join([
'{}: {}'.format(key, value) if not isinstance(value, list) else
'{}: [{}]'.format(key, ', '.join(["%.4f" % val for val in value]))
for key, value in result_in_dict.items()
])
print(result_in_str)
logging.info(result_in_str)
return result_in_dict
def test(cf, logger, max_fold=None):
"""performs testing for a given fold (or held out set). saves stats in evaluator.
"""
logger.time("test_fold")
logger.info('starting testing model of fold {} in exp {}'.format(
cf.fold, cf.exp_dir))
net = model.net(cf, logger).cuda()
batch_gen = data_loader.get_test_generator(cf, logger)
test_predictor = Predictor(cf, net, logger, mode='test')
test_results_list = test_predictor.predict_test_set(
batch_gen,
return_results=not hasattr(cf, "eval_test_separately")
or not cf.eval_test_separately)
if test_results_list is not None:
test_evaluator = Evaluator(cf, logger, mode='test')
test_evaluator.evaluate_predictions(test_results_list)
test_evaluator.score_test_df(max_fold=max_fold)
mins, secs = divmod(logger.get_time("test_fold"), 60)
h, mins = divmod(mins, 60)
t = "{:d}h:{:02d}m:{:02d}s".format(int(h), int(mins), int(secs))
logger.info('Testing of fold {} took {}.'.format(cf.fold, t))
def update_prediction_convergence(static, inputs, models):
if (inputs is not None) and (models is not None):
# Filter inputs
file = models
inputs = sorted(inputs)
# Make filename
for parameter in inputs:
temp = parameter.strip(" ")
file += "_" + temp
# Initialise model
model = Predictor(dataset, inputs=inputs, model=models, filename=file)
# Run
results = model.check()
return {
'data': [{
'x': results[-1],
'y': results[1],
'line': dict(color='black')
}]
}
def dump():
predictor = Predictor()
tf_word_feature = predictor.forward_allword()
sess = melt.load(FLAGS.model_dir)
word_feature = sess.run(tf_word_feature)
np.save(FLAGS.data_dir + '/word_feature.npy', word_feature)
sess.close()
def experiment_1():
"""
This experiment tries to predict the values of IVOL from the data by directly looking at the raw variables.
"""
# Instance from Predictor Class
p = Predictor(len(experiment_features), 32, len(target_label),
1).to(device)
# Stochastic Gradient Descent optimizer
optimizer = torch.optim.SGD(p.parameters(),
lr=lr,
momentum=learning_momentum)
# Temporary variables for computing the average loss
all_loss = 0.0
all_count = 0
# range from 1968 to 2019
dt = tqdm(
get_stock_data(experiment_train_start_year, experiment_train_end_year))
optimizer.zero_grad()
for X, Y in dt:
# Prediction
_, loss = p(X, Y)
all_loss += loss.item()
all_count += 1
loss.backward()
if all_count and all_count % cumulate_loss == 0:
nn.utils.clip_grad_norm_(p.parameters(), max_grad_norm)
optimizer.step()
optimizer.zero_grad()
dt.set_description("Average Loss: {:.2f}".format(all_loss / all_count))
def send_file():
global predictor
if predictor is None:
predictor = Predictor()
result = ''
if 'file2upload' not in request.files:
print('No file part')
return redirect(request.url)
fileob = request.files["file2upload"]
if fileob.filename == '':
print('No selected file')
return redirect(request.url)
if fileob and allowed_file(fileob.filename):
filename = secure_filename(fileob.filename)
#result = predictor.predict(fileob)
#Once we want to store the files on the server
save_path = os.path.join(app.config["UPLOAD_FOLDER"], filename)
fileob.save(save_path)
with open(save_path, "r") as f:
result = predictor.predict(f)
#try:
# os.remove(save_path)
#except Exception as error:
# app.logger.error("Error deleting file after processing", error)
return result
def model_fn(model_dir):
"""
Loads the model from disk and return a model object
:param model_dir: The directory in which the model is located
:return: Model object
"""
return Predictor(model_dir)
def setupUi(self, MainWindow):
MainWindow.setObjectName("Homework")
MainWindow.resize(414, 285)
self.centralWidget = QtWidgets.QWidget(MainWindow)
self.centralWidget.setObjectName("centralWidget")
self.scoreButton = QtWidgets.QPushButton(self.centralWidget)
self.scoreButton.setGeometry(QtCore.QRect(280, 120, 89, 25))
self.scoreButton.setObjectName("scoreButton")
self.resultLabel = QtWidgets.QLabel(self.centralWidget)
self.resultLabel.setGeometry(QtCore.QRect(180, 203, 64, 64))
self.resultLabel.setObjectName("resultLabel")
self.widget = QtWidgets.QWidget(self.centralWidget)
self.widget.setGeometry(QtCore.QRect(30, 30, 201, 171))
self.widget.setObjectName("widget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.widget)
self.verticalLayout.setContentsMargins(11, 11, 11, 11)
self.verticalLayout.setSpacing(6)
self.verticalLayout.setObjectName("verticalLayout")
self.label = QtWidgets.QLabel(self.widget)
self.label.setObjectName("label")
self.verticalLayout.addWidget(self.label)
self.tweetTextEdit = QtWidgets.QPlainTextEdit(self.widget)
self.tweetTextEdit.setObjectName("tweetTextEdit")
self.verticalLayout.addWidget(self.tweetTextEdit)
MainWindow.setCentralWidget(self.centralWidget)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
# non-ui
self.setupEvents()
self.predictor = Predictor("models/predictor.pkl")
def eval_model(args) -> None:
predictor = Predictor(args.archive_file, args.cuda_device,
args.predicted_pages, args.merge_google,
args.score_format, args.verbose)
raw_data = []
with open(args.in_file) as f:
for line in f:
raw_data.append(json.loads(line))
actual = []
predicted = []
if args.log is not None:
f = open(args.log, "w+")
#print({util.get_device_of(param) for param in model.parameters()})
for output in predictor.predict(raw_data[args.start:args.end]):
actual.append(output['actual'] if 'actual' in
output else output.get('label', 'NOT ENOUGH INFO'))
predicted.append(output['predicted_label'] if 'predicted_label' in
output else output['predicted'])
if args.log is not None:
f.write(json.dumps(output) + "\n")
if args.log is not None:
f.close()
if args.verbose:
print(accuracy_score(actual, predicted))
print(classification_report(actual, predicted))
print(confusion_matrix(actual, predicted))
def callback(self, ch, method, properties, body):
print(" [x] Received %r" % body)
'''
model predict code here.
write to csv
'''
predictor = Predictor()
face_img_path = body
if not os.path.exists(face_img_path):
print("face path not exist.")
return
img_name = str(body).split('\\')[-1]
# print(img_name)
with open(face_img_path, 'rb') as f:
image_bytes = f.read()
print("inference:")
start_time = time.time()
label = predictor.predict(image_bytes, img_name)
end_time = time.time()
inference_latency = end_time - start_time
print("label = " + str(label) + ", " + str(inference_latency))
print(" Run model predict here. \n")
def main():
teams = []
args = arguments()
predictor = Predictor(args.dataset)
for team in Teams():
teams.append(Team(team.name, team.abbreviation))
parse_boxscores(predictor, teams, args.skip_save_to_mongodb)
def predict_tips():
# Validate the access token
authorization = request.headers.get('Authorization', None)
if authorization is None or authorization.split(' ')[1] != os.environ.get(
'ACCESS_TOKEN'):
return {'error': 'Please provide a valid access token'}, 400
# Retrieve the request body
req_body = request.get_json()
if req_body is None:
return {'error': 'Please provide necessary parameters'}, 400
try:
# Parse the request body to retreive required parameters
user_id, search_dates = parse_request_body(req_body)
except Exception as err:
print(err)
return {'error': str(err)}, 400
try:
# Get predicted tips for each specified day
predictor = Predictor(user_id=user_id, search_dates=search_dates)
return predictor.get_predicted_tips(), 200
except DatasetError as err:
return {'error': str(err)}, 500
except Exception as err:
print(err)
return {
'error':
'Failed to predict tips for the user. Please try again later'
}, 500
def predict(self, args: ClassifierArgs, **kwargs):
# self.evaluate(args, is_training=False)
self.loading_model_from_file(
args.saving_dir, args.build_saving_file_name(description='best'))
self.model.eval()
predictor = Predictor(self.model, self.data_processor, args.model_type)
dataset, _ = self.build_data_loader(args,
args.evaluation_data_type,
tokenizer=False)
assert isinstance(dataset, ListDataset)
if args.predict_numbers == -1:
predict_dataset = dataset.data
else:
predict_dataset = np.random.choice(dataset.data,
size=(args.predict_numbers, ),
replace=False)
description = tqdm(predict_dataset)
metric = RandomSmoothAccuracyMetrics()
for data in description:
tmp_instances = self.mask_instance_decorator(
args, data, args.predict_ensemble)
tmp_probs = predictor.predict_batch(tmp_instances)
target = self.data_reader.get_label_to_idx(data.label)
pred = predict(tmp_probs, args.alpha)
metric(pred, target)
description.set_description(metric.__str__())
print(metric)
logging.info(metric)
def evaluate():
reader = PWKPReader()
vocab = Vocabulary.from_files(vocab_dir)
iterator = BasicIterator(batch_size=opt.batch_size)
iterator.index_with(vocab)
model = Seq2Seq(emb_size=opt.emb_size,
hidden_size=opt.hidden_size,
enc_layers=opt.enc_layers,
dec_layers=opt.dec_layers,
dropout=opt.dropout,
bidirectional=opt.bidirectional,
beam_size=opt.beam_size,
label_smoothing=opt.label_smoothing,
vocab=vocab)
model = model.cuda(opt.gpu)
model_state = torch.load(opt.restore, map_location=util.device_mapping(-1))
model.load_state_dict(model_state)
predictor = Predictor(iterator=iterator,
max_decoding_step=opt.max_step,
vocab=vocab,
reader=reader,
data_path=test_path,
log_dir=save_dir,
map_path=ner_path,
cuda_device=opt.gpu)
predictor.evaluate(model)
def main(args):
config_path = os.path.join(args.model_dir, 'config.json')
with open(config_path) as f:
config = json.load(f)
logging.info('loading embedding...')
with open(config['model']['embedding'], 'rb') as f:
embedding = pickle.load(f)
config['model']['embedding'] = embedding.vectors
logging.info('loading valid data...')
with open(config['model']['valid'], 'rb') as f:
config['model']['valid'] = pickle.load(f)
logging.info('loading train data...')
with open(config['train'], 'rb') as f:
train = pickle.load(f)
predictor = Predictor(arch=config['arch'],
device=args.device,
metrics=[Recall()],
**config['model'])
if args.load is not None:
predictor.load(args.load)
model_checkpoint = ModelCheckpoint(
os.path.join(args.model_dir, 'model.pkl'), **config['callbacks'])
metrics_logger = MetricsLogger(os.path.join(args.model_dir, 'log.json'))
logging.info('start training!')
predictor.fit_dataset(train, train.collate_fn, model_checkpoint,
metrics_logger)
def main():
args = parse_arguments()
predictor = Predictor()
simulation = load_simulation()
seeds = find_projected_seeds(simulation, args.conference)
winner = simulate_tournament(seeds, BRACKETS[args.conference], predictor)
print winner
def get_test_prediction(data_value_arrays, labels, model):
p = Predictor(model,
file='/tmp/predict_gbm.log',
loglevel=logging.DEBUG)
y_predicted_df = p.predict(data_value_arrays, labels)
rmse = p.score(data_value_arrays, labels)
return rmse, y_predicted_df
def test(logger):
"""
perform testing for a given fold (or hold out set). save stats in evaluator.
"""
logger.info('starting testing model of fold {} in exp {}'.format(
cf.fold, cf.exp_dir))
net = model.net(cf, logger).cuda()
test_predictor = Predictor(cf, net, logger, mode='test')
test_evaluator = Evaluator(cf, logger, mode='test')
batch_gen = data_loader.get_test_generator(cf, logger)
test_results_list, test_results_list_mask, test_results_list_seg, test_results_list_fusion, testing_epoch = test_predictor.predict_test_set(
batch_gen, cf, return_results=True)
count = test_evaluator.evaluate_predictions(test_results_list,
testing_epoch,
cf,
pth=cf.test_dir,
flag='test')
print('tp_patient {}, tp_roi {}, fp_roi {}'.format(count[0], count[1],
count[2]))
save_test_image(test_results_list, test_results_list_mask,
test_results_list_seg, test_results_list_fusion,
testing_epoch, cf, cf.plot_dir)
def train_epoches(t_dataset, v_dataset, model, n_epochs,
teacher_forcing_ratio):
eval_f = Evaluate_test()
best_dev = 0
train_loader = t_dataset.corpus
len_batch = len(train_loader)
epoch_examples_total = t_dataset.len
for epoch in range(1, n_epochs + 1):
model.train(True)
torch.set_grad_enabled(True)
epoch_loss = 0
for batch_idx in range(len_batch):
loss, num_examples = train_batch(t_dataset, batch_idx, model,
teacher_forcing_ratio)
epoch_loss += loss * num_examples
sys.stdout.write('%d batches processed. current batch loss: %f\r' %
(batch_idx, loss))
sys.stdout.flush()
epoch_loss /= epoch_examples_total
log_msg = "Finished epoch %d with losses: %.4f" % (epoch, epoch_loss)
print(log_msg)
predictor = Predictor(model, v_dataset.vocab, args.cuda)
print("Start Evaluating")
cand, ref = predictor.preeval_batch(v_dataset)
print('Result:')
print('ref: ', ref[1][0])
print('cand: ', cand[1])
final_scores = eval_f.evaluate(live=True, cand=cand, ref=ref)
epoch_score = 2 * final_scores['ROUGE_L'] * final_scores['Bleu_4'] / (
final_scores['Bleu_4'] + final_scores['ROUGE_L'])
if epoch_score > best_dev:
torch.save(model.state_dict(), args.save)
print("model saved")
best_dev = epoch_score
def main(args):
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
model_dir, exp_dir = os.path.split(
args.output_dir[:-1]) if args.output_dir[-1] == '/' else os.path.split(
args.output_dir)
config_path = os.path.join(model_dir, 'config.json')
with open(config_path) as f:
config = json.load(f)
logging.info(f'Save config file to {args.output_dir}.')
with open(os.path.join(args.output_dir, 'config.json'), 'w') as f:
json.dump(config, f, indent=4)
logging.info('Loading training data...')
with open(config['train_path'], 'rb') as f:
train = pickle.load(f)
train.context_padded_len = config['train_context_padded_len']
train.option_padded_len = config['train_option_padded_len']
train.n_negative = config['train_n_negative']
logging.info('Loading validation data...')
with open(config['valid_path'], 'rb') as f:
valid = pickle.load(f)
config['model_parameters']['valid'] = valid
valid.context_padded_len = config['valid_context_padded_len']
valid.option_padded_len = config['valid_option_padded_len']
logging.info('Loading preproprecessed word embedding...')
with open(config['embedding_path'], 'rb') as f:
embedding = pickle.load(f)
config['model_parameters']['embedding'] = embedding
metric = Recall(at=10)
predictor = Predictor(training=True,
metrics=[metric],
device=args.device,
**config['model_parameters'])
model_checkpoint = ModelCheckpoint(filepath=os.path.join(
args.output_dir, 'model'),
monitor=metric.name,
mode='max',
all_saved=False)
metrics_logger = MetricsLogger(
log_dest=os.path.join(args.output_dir, 'log.json'))
if args.load_dir is not None:
predictor.load(args.load_dir)
logging.info('Start training.')
start = time.time()
predictor.fit_dataset(data=train,
collate_fn=train.collate_fn,
callbacks=[model_checkpoint, metrics_logger],
output_dir=args.output_dir)
end = time.time()
total = end - start
hrs, mins, secs = int(total // 3600), int(
(total % 3600) // 60), int(total % 60)
logging.info('End training.')
logging.info(f'Total time: {hrs}hrs {mins}mins {secs}secs.')
def main(_):
if FLAGS.is_train:
if os.path.exists(FLAGS.checkpoint_dir):
shutil.rmtree(FLAGS.checkpoint_dir)
if os.path.exists('logs'):
shutil.rmtree('logs')
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#config.gpu_options.per_process_gpu_memory_fraction = 0.8
with tf.Session(config=config) as sess:
predictor = Predictor(sess,
batch_size=FLAGS.batch_size,
checkpoint_dir=FLAGS.checkpoint_dir,
n_hidden=FLAGS.n_hidden,
Kx=FLAGS.Kx,
Ky=FLAGS.Ky,
clean=FLAGS.clean,
first=FLAGS.first)
if FLAGS.is_train:
predictor.train(FLAGS)
else:
if predictor.load(FLAGS.checkpoint_dir):
predictor.predict(FLAGS.test_data)
def main(args):
config_path = os.path.join(args.model_dir, 'config.json')
with open(config_path) as f:
config = json.load(f)
logging.info('loading valid data...')
with open(config['model']['valid'], 'rb') as f:
config['model']['valid'] = pickle.load(f)
logging.info('loading character vocabulary...')
with open(config['charmap'], 'rb') as f:
charmap = pickle.load(f)
logging.info('loading word vocabulary...')
with open(config['wordmap'], 'rb') as f:
wordmap = pickle.load(f)
config['model']['num_embeddings'] = len(charmap)
config['model']['padding_idx'] = charmap['<PAD>']
config['model']['vocab_size'] = len(wordmap)
predictor = Predictor(arch=config['arch'],
device=args.device,
metrics=[Perplexity()],
**config['model'])
if args.load is not None:
predictor.load(args.load)
model_checkpoint = ModelCheckpoint(
os.path.join(args.model_dir, 'model.pkl'), **config['callbacks'])
metrics_logger = MetricsLogger(os.path.join(args.model_dir, 'log.json'))
logging.info('start training!')
predictor.fit_dataset(config['train'], model_checkpoint, metrics_logger)
def do_predictor_rollout(predictor_args, initial_state, act_list, noise_list,
step):
"""act_list: list with num_rollout_per_cpu elements, each element is np.array with size (H, dim_u)"""
# print('begin rollout...')
predictor = Predictor(*predictor_args)
print('predictor built successfully')
paths = []
N = len(act_list)
H = act_list[0].shape[0]
for i in range(N):
predictor.catch_up(*initial_state)
act = []
noise = []
obs = []
cost = []
for k in range(H):
obs.append(predictor._get_obs())
act.append(act_list[i][k])
noise.append(noise_list[i][k])
c = predictor.predict(act[-1], step)
cost.append(c)
path = dict(observations=np.array(obs),
actions=np.array(act),
costs=np.array(cost),
noise=np.array(noise))
paths.append(path)
return paths
class Controller:
__app = web.Application()
__predictor = Predictor()
@classmethod
async def run_app(cls):
"""
Route the requests to the methods that can handle them
:return: __app
"""
cls.__app.add_routes([web.post("/predict", cls.handle)])
return cls.__app
@classmethod
async def handle(cls, request: web.Request):
"""
Accept the photo, parse into acceptable format and make a prediction
:return: json
"""
post = await request.post()
image = post.get('image')
if image:
img_content = image.file.read()
output = await cls.__predictor.predict(img_content)
return web.json_response({"melanoma probability": str(output)})
