def evaluate(valloader, model, criterion, save_image):
# switch the model to evaluate mode ( important for dropout layers or batchnorm layers )
model.evaluate()
loss_sum = 0
for i, data in enumerate(valloader):
# get train and label data
val, label = data
# put on gpu or cpu
val = val.to(device)
# label is of type TensorLong
label = label.to(device)
# for the CrossEntropyLoss:
# outputs needs to be of size: Minibatch, classsize, dim 1, dim 2 , ...
# outputs are 2 classes with 2d images. Channelsize = class size
# label needs to be of format: Minibatch, dim 1, dim 2, ...
# i cut the channel info for it to work, because it is only a 2d image.
# as an alternative, one could add 1 channel for class in train, than label does not need any change
# label normally looks like: ( minibatchsize, 1, width, height )
label = label.view(label.size(0), label.size(2), label.size(3))
# forward + backward + optimize
outputs = model(val, padding=args.pad)
loss = criterion(torch.log(outputs), label)
running_loss = loss.item()
loss_sum = loss_sum + running_loss
if save_image:
save_images_eval(outputs, './eval/', index=i + 1)
del outputs, val, label, loss
loss_avg = loss_sum / (i + 1)
return loss_avg
def run():
print('loading training data...')
X, y = load_training_data()
#min_max_scaler = preprocessing.MinMaxScaler()
#X = min_max_scaler.fit_transform(X)
util.count(y)
print('loading data completed.')
print('loading models...')
models = md.getClassifiers()
print('training models...')
md.train(models, X, y)
print('training models completed...')
print('loading test data...')
X_test, y_test = load_test_data()
#min_max_scaler = preprocessing.MinMaxScaler()
#X_test = min_max_scaler.fit_transform(X_test)
util.count(y_test)
print('loading test data completed...')
print('eval models...')
md.evaluate(models, X_test, y_test)
print('eval models completed...')
def pred(data, countries, start):
# ***** Get predictions for all country *****#
open('predictions.txt', 'w').close()
for country in countries:
record_predictions('\n' + country + ':')
print('\n', country, ':')
df = data[['Week', country]]
df = df[np.isfinite(df[country])] # remove empty rows
df = df.rename(index=str, columns={country: "Now"})
for i in range(3):
df_ = df.copy()
df_ = prepare(df_, pred=True)
# train the model using all data except for the latest record
X = df_.iloc[:,:-1]
y = df_.iloc[:,-1]
X,y = model.preprocess_data(X,y)
X_train, y_train = X[:-1], y[:-1]
# printX_train, '\n', y_train
lr, label = model.lr(X_train,y_train)
model.evaluate(lr,X,y,X_train,X_train,y_train,y_train,label,graph=False)
fea = X[-1]
y_pred = lr.predict([fea])
print('features:', fea, '\npredictions:', y_pred[0])
record_predictions(str(y_pred[0]))
rec = pd.DataFrame([[start+i, y_pred[0]]], columns=['Week','Now'])
# print'\nbefore appending\n', df
df = df.append(rec)
def main():
###############################################################################
# Load data
###############################################################################
d = util.Dictionary()
if args.task == "train":
logging.info("Reading train...")
trncorpus = util.read_corpus(args.ftrn, d, True)
d.freeze() # no new word types allowed
vocab_size = d.size()
# save dict
d.save_dict(fprefix + ".dict")
logging.info("Reading dev...")
devcorpus = util.read_corpus(args.fdev, d, False)
elif args.task == "test":
logging.info("Reading test...")
d.load_dict(args.fdct)
d.freeze()
vocab_size = d.size()
# load test corpus
tstcorpus = util.read_corpus(args.ftst, d, False)
###############################################################################
# Build the model
###############################################################################
if args.task == "train":
model_fname = fprefix + ".model"
pretrained_model = None
if args.fmod:
# load pre-trained model
pretrained_model = model.load_model(args.fmod, vocab_size,
args.nclass, args.inputdim,
args.hiddendim, args.nlayer,
args.droprate)
logging.info("Successfully loaded pretrained model.")
trained = model.train(trncorpus, devcorpus, vocab_size, args.nclass,
args.inputdim, args.hiddendim, args.nlayer,
args.trainer, args.lr, args.droprate, args.niter,
args.logfreq, args.verbose, model_fname,
pretrained_model)
dev_accuracy = model.evaluate(trained, devcorpus.docs)
logging.info("Final Accuracy on dev: %s", dev_accuracy)
model.save_model(trained, model_fname)
else:
trained_model = model.load_model(args.fmod, vocab_size, args.nclass,
args.inputdim, args.hiddendim,
args.nlayer, args.droprate)
tst_accuracy = model.evaluate(trained_model, tstcorpus.docs)
logging.info("Final Accuracy on test: %s", tst_accuracy)
def main(config):
data_gainer = Data(config)
train_x, train_y = data_gainer.get_train_data()
valid_x, valid_y = data_gainer.get_valid_data()
test_x = data_gainer.get_test_data()
train_label = data_gainer.return_label("train")
valid_label = data_gainer.return_label("valid")
test_label = data_gainer.return_label("test")
# train
_, tuning_loss_mean = train(train_x, train_y, config)
loss_plot(tuning_loss_mean)
# evaluate
y_pred_valid = predict(valid_x, config)
evaluate(y_pred_valid, valid_label, data_gainer)
def wave(num):
"""
The basic calculating and drawing function.
The internal function is 'cosine' >> (x, y) values.
The function uses slider values to variate the output.
Slider values are defined by <slider name>.val
"""
for i, letter in enumerate('abcdefghijklmnopqrstuvwxyz'):
evaluate(i, np.array([s.val for s in slides]))
sprite = pygame.image.load('out.png')
sprite = pygame.transform.scale2x(sprite)
rect = sprite.get_rect()
rect.center = (50 + i % 13 * 62, 50 + i//13 * 90)
screen.blit(sprite, rect)
def run_training(train_file, base_path, device, bidirectional=False):
corpus_name = os.path.split(train_file)[1].replace('.train', '')
print(f"Training with {corpus_name}")
model_name = corpus_name + ("_LSTM" if not bidirectional else "_BiLSTM")
model_path = base_path.joinpath(model_name)
model_path.mkdir(exist_ok=True)
dev_file = train_file.replace(".train", ".dev")
test_file = train_file.replace(".train", ".test")
print(f"Loading {train_file}")
train_data = EosDataset(train_file, split_dev=False, min_freq=10000, save_vocab=model_path.joinpath("vocab.pickle"),
remove_duplicates=False, shuffle_input=False)
print(f"Loading {dev_file}")
dev_data = EosDataset(dev_file, split_dev=False, min_freq=10000, load_vocab=model_path.joinpath("vocab.pickle"),
remove_duplicates=False, shuffle_input=False)
print(f"Loading {test_file}")
test_data = EosDataset(test_file, split_dev=False, min_freq=10000, load_vocab=model_path.joinpath("vocab.pickle"),
remove_duplicates=False, shuffle_input=False)
model = DeepEosModel(max_features=20000, rnn_bidirectional=bidirectional)
model.to(device)
print(model)
print(f"Number of trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)}")
optimizer = optim.Adam(model.parameters(), lr=0.001)
print("Starting Traning")
train(model, train_data, dev_data, base_path=model_path, optimizer=optimizer, epochs=5, device=device)
print("Evaluating")
with open(model_path.joinpath("evaluation.txt"), 'w', encoding='UTF-8') as f:
precision, recall, f1, accuracy = evaluate(model, test_data, device=device)
f.write(f"Precision: {precision}\nRecall: {recall}\nF1: {f1}\nAccuracy: {accuracy}\n")
def test():
model = edsr(scale=scale, num_res_blocks=depth, num_filters=channels)
checkpoint = tf.train.Checkpoint(step=tf.Variable(0),
psnr=tf.Variable(-1.0),
optimizer=Adam(1e-04),
model=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint=checkpoint,
directory='./ckpt',
max_to_keep=3)
restore(checkpoint, checkpoint_manager)
video_valid = video_ds(subset='valid')
valid_ds = video_valid.dataset(batch_size=1,
random_transform=False,
repeat_count=1)
psnr, ssim = evaluate(checkpoint.model, valid_ds)
print('PSNR:%.3f, SSIM:%.3f' % (psnr, ssim))
psnr_b = bilinear_upscale('../image_240', '../image_960', scale=scale)
print('bilinear upscale PSNR:%.3f' % psnr_b)
lr = load_image('../image_240/frame1500.jpg')
sr = resolve_single(checkpoint.model, lr)
plt.imshow(sr)
plt.show()
def __evaluate(self, sess, model, name, data, idToTag, logger):
evaluateResult = dict()
logger.info("evaluate:{}".format(name))
nerResults = model.evaluate(sess, data,
idToTag) # 预测结果，格式【原始字符 正确标注 预测标注结果】
evalResult, evalResultDict = self.curDep.testNer(
nerResults, self.FLAGS.resultPath)
for line in evalResult: #打印出来
logger.info(line)
wholeDict = evalResultDict['wholeClass']
subClassDict = evalResultDict["subClass"]
if name == 'dev':
evaluateResult['TrainWhole'] = wholeDict
evaluateResult['TrainSubclass'] = subClassDict
if name == 'test':
evaluateResult['DevWhole'] = wholeDict
evaluateResult['DevSubclass'] = subClassDict
f1 = evalResultDict["wholeClass"]['MacroF1']
if name == "dev":
bestDevF1 = model.bestDevF1.eval()
if f1 > bestDevF1:
tf.assign(model.bestDevF1,
f1).eval() # tf.assign()更新赋值,把best更新为f1.
logger.info("new best dev f1 score:{:>.3f}".format(f1))
return f1 > bestDevF1, evaluateResult
elif name == "test":
bestTestF1 = model.bestTestF1.eval()
if f1 > bestTestF1:
tf.assign(model.bestTestF1, f1).eval()
logger.info("new best test f1 score:{:>.3f}".format(f1))
return f1 > bestTestF1, evaluateResult
def train(model, training_data, callback=True, batch_size=256, epochs=10):
(x_train, y_train), (x_test, y_test), mapping, nb_classes = training_data
# convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
if callback == True:
# Callback for analysis in TensorBoard
tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True)
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test),
callbacks=[tbCallBack] if callback else None)
score = model.evaluate(x_test, y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
# Offload model to file
model_yaml = model.to_yaml()
with open("bin/model.yaml", "w") as yaml_file:
yaml_file.write(model_yaml)
save_model(model, 'bin/model.h5')
def main(data, model):
print("Reading images...")
x_train = data.read_images('./data/trainImage256.txt', TRAIN_DATA_SIZE)
x_test = data.read_images('./data/testImage256.txt', TEST_DATA_SIZE)
y_train = data.read_labels('./data/trainLABEL256.txt', TRAIN_DATA_SIZE)
y_test = data.read_labels('./data/testLABEL256.txt', TEST_DATA_SIZE)
print("Creating model...")
model.create_model(multi_gpu=False)
print("Now training...")
history = model.training(x_train, y_train, x_test, y_test)
accuracy = history.history["accuracy"]
loss = history.history["loss"]
eval = model.evaluate(x_test, y_test)
print("accuracy = " + str(eval))
model.save('./model.h5')
if not os.path.exists('./result_keras'):
os.mkdir('./result_keras')
for i in range(TEST_DATA_SIZE):
ret = model.predict(x_test[i, :, :, 0].reshape([1, IMG_SIZE, IMG_SIZE, 1]), 1)
np.savetxt('./result_keras/' + str(i) + '.txt', ret[0, :, :, 0])
with open("training_log.txt", "w") as f:
for i in range(training_epochs):
f.write(str(loss[i]) + "," + str(accuracy[i]) + "\n")
ax1 = plt.subplot()
ax1.plot(loss, color="blue")
ax2 = ax1.twinx()
ax2.plot(accuracy, color="orange")
plt.show()
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
if self._step % FLAGS.eval_frequency == 0:
precision = model.evaluate(run_context.session,
top_k_op,
FLAGS.num_valid_examples)
format_str = (
'%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch), precision = %.2f%%')
print(format_str %
(datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch, precision))
else:
format_str = (
'%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str %
(datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
def evaluate(self, model_pickle, test_data) -> dict:
print("Evaluating model")
model = pickle.loads(model_pickle)
score = model_package.evaluate(model, test_data)
return {"SMAPE": score}
def inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS):
if FLAGS.max_steps == -1:
FLAGS.max_steps = 1000
if FLAGS.saved_model_input_path:
cast_dtype = tf.float16 if FLAGS.amp else tf.float32
else:
cast_dtype = None
auc, test_loss, latencies = evaluate(validation_pipeline, dlrm,
timer, auc_thresholds=FLAGS.auc_thresholds,
data_parallel_splitter=splitter,
max_steps=FLAGS.max_steps, cast_dtype=cast_dtype)
# don't benchmark the first few warmup steps
latencies = latencies[10:]
result_data = {
'mean_inference_throughput': FLAGS.valid_batch_size / np.mean(latencies),
'mean_inference_latency': np.mean(latencies)
}
for percentile in [90, 95, 99]:
result_data[f'p{percentile}_inference_latency'] = np.percentile(latencies, percentile)
result_data['auc'] = auc
if hvd.rank() == 0:
dllogger.log(data=result_data, step=tuple())
def check_and_log(self, learning, dataset, sz):
if self.conditional_log is None: return
if sz == 0 or sz >= self.conditional_log[0]:
test_err = MModel.evaluate(learning.model, dataset.test_data)
add_to_dict(self.err_log, sz,
[np.array([learning.cnt_labels, test_err])])
if sz >= self.conditional_log[0]:
self.conditional_log[0] *= self.conditional_log[1]
def evaluateAndShowAttention(input_sentence, input_lang, output_lang, encoder1,
attn_decoder1):
output_words, attentions = model.evaluate(encoder1, attn_decoder1,
input_sentence, input_lang,
output_lang)
print('input =', input_sentence)
print('output =', ' '.join(output_words))
showAttention(input_sentence, output_words, attentions)
def fine_tune(model: Union[DeepEosModel, str], vocab_path: Union[str, Path], cross_validation_set=None):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
window_size = 6
if type(model) is str:
model = torch.load(str(model), map_location=device)
# BIOfid fine-tuning
biofid_train = EosDataset('data/bioFID_train_cleaned.txt', split_dev=False, window_size=window_size,
load_vocab=vocab_path, shuffle_input=False)
biofid_test = EosDataset('data/bioFID_test.txt', split_dev=False, window_size=window_size, load_vocab=vocab_path,
shuffle_input=False)
print("\nBIOfid test scores prior to fine-tuning")
evaluate(model, biofid_test, device=device)
print(flush=True)
optimizer = optim.Adam(model.parameters(), lr=0.0001)
train(model, biofid_train, biofid_test, base_path=Path('biofid_models/'),
optimizer=optimizer, epochs=10, device=device)
print("\nBIOfid test")
evaluate(model, biofid_test, device=device)
if cross_validation_set is not None:
print("\nCross validation")
evaluate(model, cross_validation_set, device=device)
model_name = 'de.biofid'
torch.save(model, Path('biofid_models/').joinpath(model_name + '.pt'))
def aggregate():
if len(selected_clients) <= 0:
return {"success": False}
print("Waiting for updates...")
updates = []
global version
selected_clients_copy = selected_clients.copy()
while len(selected_clients_copy) > 0:
update = aggregator.recv_pyobj()
print("Recieved update %d/%d" %
(len(selected_clients_copy), len(selected_clients)))
client_id = update["client_id"]
model_version = update["version"]
client_metrics = update["metrics"]
if client_id in selected_clients_copy and model_version == version:
print("Received update on version %s from client %s" %
(model_version, client_id))
print("Metrics: %s" %
json.dumps(client_metrics, indent=4, sort_keys=True))
updates.append(update)
selected_clients_copy.remove(client_id)
total_points = 0
for update in updates:
total_points += update["points"]
print(total_points)
weighted_avg = np.array(
[np.zeros(layer.shape) for layer in model.get_weights()])
for update in updates:
points = update["points"]
weights = update["weights"]
weighted_avg += (points / total_points) * np.array(weights)
model.set_weights(weighted_avg.tolist())
version += 1
print("Current version: %s" % version)
print("Evaluating...")
history = model.evaluate(x=x_test, y=y_test, batch_size=32)
return {"success": True, "loss": history[0], "accuracy": history[1]}
def main(args):
print("Loading Dataset...")
dataset = testset_dataloader(args.tokenizer, args)
print("Dataset Loaded...")
print(f"Dataset size: {len(dataset.dataset)}")
pred = evaluate(args.model, dataset, args)
result = pd.DataFrame(pred)
result.index += 1
result.to_csv(args.output_path, header=False)
print(f"Results successfully saved at {args.output_path}")
async def echo(message):
# media = types.MediaGroup()
WIDTH = message.photo[-1].width
HEIGHT = message.photo[-1].height
byteImgIO = BytesIO()
await message.photo[-1].download(byteImgIO)
byteImgIO.seek(0)
byteImg = byteImgIO.read()
dataBytesIO = BytesIO(byteImg)
# im = Image.frombuffer('RGB', (WIDTH, HEIGHT), dataBytesIO.getvalue())
im = np.array(Image.open(dataBytesIO))
res_l = evaluate(im)
res_l = [x for x in res_l if x not in ['<unk>', '<end>']]
result = ' '.join(res_l)
result_ru = translate(result)
print('result: ', result, result_ru)
try:
r = requests.post('https://ttsmp3.com/makemp3_new.php',
data={
'msg': result_ru,
'lang': 'Maxim',
'source': 'ttsmp3'
},
headers={'User-agent': 'Mozilla/5.0'},
config={'verbose': sys.stderr})
print(r)
print(r.json())
r1 = requests.get(r.json()['URL'])
with open('./audio/audio.ogg', 'wb') as f:
f.write(r1.content)
except:
var = gTTS(text=result_ru, lang='ru')
var.save('./audio/audio.ogg')
await message.answer_voice(types.InputFile('./audio/audio.ogg'))
await message.answer("''" + result_ru + "''")
await message.answer(__text)
def inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS):
if FLAGS.max_steps == -1:
FLAGS.max_steps = 1000
_, _, latencies = evaluate(validation_pipeline,
dlrm,
timer,
auc_thresholds=None,
data_parallel_splitter=splitter,
max_steps=FLAGS.max_steps)
# don't benchmark the first few warmup steps
latencies = latencies[10:]
result_data = {
'mean_inference_throughput':
FLAGS.valid_batch_size / np.mean(latencies),
'mean_inference_latency': np.mean(latencies)
}
for percentile in [90, 95, 99]:
result_data[f'p{percentile}_inference_latency'] = np.percentile(
latencies, percentile)
dllogger.log(data=result_data, step=tuple())
def test(self):
"""Computing AUC for the unseen test set """
# Load trained parameters
G_path = os.path.join(self.model_save_path,
'{}_G.pth'.format(self.test_model))
self.device = torch.device("cuda:0")
self.G.load_state_dict(torch.load(G_path))
self.G.eval()
data_loader = self.img_data_loader
# Create big error tensor for the test set.
self.an_scores = torch.zeros(size=(len(data_loader.valid.dataset), ),
dtype=torch.float32,
device=self.device)
self.gt_labels = torch.zeros(size=(len(data_loader.valid.dataset), ),
dtype=torch.long,
device=self.device)
with torch.no_grad():
for i, (real_x, org_c) in enumerate(data_loader.valid):
real_x = self.to_var(real_x, volatile=True)
fake_x, enc_feat, rec_feat = self.G(real_x)
error = torch.mean(
torch.pow(
(enc_feat - rec_feat), 2), dim=(1, 2, 3)) + torch.mean(
torch.abs(real_x - fake_x), dim=(1, 2, 3))
self.an_scores[i * self.batch_size:i * self.batch_size +
error.size(0)] = error.reshape(error.size(0))
self.gt_labels[i * self.batch_size:i * self.batch_size +
error.size(0)] = org_c.reshape(error.size(0))
self.an_scores = (self.an_scores - torch.min(self.an_scores)) / (
torch.max(self.an_scores) - torch.min(self.an_scores))
self.an_scores = self.an_scores.detach()
self.an_scores = self.an_scores.cpu().numpy()
self.gt_labels = self.gt_labels.detach()
self.gt_labels = self.gt_labels.cpu().numpy()
auc = evaluate(self.gt_labels, self.an_scores, metric='roc')
import data import model import korali k = korali.Engine() e = korali.Experiment() # Defining Concurrent Jobs popSize = 512 if (len(sys.argv) > 1): popSize = int(sys.argv[1]) # Setting up the reference likelihood for the Bayesian Problem e["Problem"]["Type"] = "Bayesian/Reference" e["Problem"]["Likelihood Model"] = "Normal" e["Problem"]["Reference Data"] = data.getReferenceData().tolist() e["Problem"]["Computational Model"] = lambda koraliData: model.evaluate( koraliData, data.getReferencePoints(), korali.getMPIComm() ) e["Distributions"][0]["Name"] = "Uniform 0" e["Distributions"][0]["Type"] = "Univariate/Uniform" e["Distributions"][0]["Minimum"] = 0.2 e["Distributions"][0]["Maximum"] = 0.6 e["Distributions"][1]["Name"] = "Uniform 1" e["Distributions"][1]["Type"] = "Univariate/Uniform" e["Distributions"][1]["Minimum"] = 10.0 e["Distributions"][1]["Maximum"] = 40.0 e["Distributions"][2]["Name"] = "Uniform 2" e["Distributions"][2]["Type"] = "Univariate/Uniform" e["Distributions"][2]["Minimum"] = 1e-5 e["Distributions"][2]["Maximum"] = 2.0
'params':[par for par in to_be_trained if par in no_decay],
'weight_decay':0
}
]
#optimizer = Adam(grouped_params, lr=LR)
optimizer = AdamW(grouped_params, lr=LR)
best_valid_accuracy = 0
train_list, valid_list = [], []
for epoch in range(EPOCHS):
# tarin step over all batches
model.train_step(network, train_data_loader, loss_function, optimizer, device)
train_targets, train_outputs = model.evaluate(network, train_data_loader, device)
valid_targets, valid_outputs = model.evaluate(network, valid_data_loader, device)
train_accuracy = accuracy_score(train_targets, [int(i>0.5) for i in train_outputs])
valid_accuracy = accuracy_score(valid_targets, [int(i>0.5) for i in valid_outputs])
train_list.append(train_accuracy)
valid_list.append(valid_accuracy)
print("Train Accuracy", train_accuracy)
print("Valid Accuracy", valid_accuracy)
if valid_accuracy > best_valid_accuracy:
best_valid_accuracy = valid_accuracy
#torch.save(network.state_dict(), MODEL_PATH)
import data
import model
import display
(train_images, train_labels), (test_images, test_labels) = data.get_data()
class_names = [
'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt',
'Sneaker', 'Bag', 'Ankle boot'
]
# display.display_sample(train_images, class_names, train_labels)
model = model.get_model()
model.fit(train_images, train_labels, epochs=5)
test_loss, test_acc = model.evaluate(test_images, test_labels)
print('Test accuracy:', test_acc)
predictions = model.predict(test_images)
display.plot_results(predictions, test_images, test_labels, class_names)
stats.to_csv("data/wine_stats.csv", sep=',', encoding='utf-8')
# Step 4: Training
# Create a function that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath = model_name,
save_weights_only=True,
verbose=0, save_freq="epoch")
# model.load_weights(model_name)
model.fit(X_train, Y_train,
batch_size = batch_size,
epochs = epochs,
validation_data= (X_valid,Y_valid),
callbacks = [cp_callback])
# Step 6: Evaluation
mae, mse = model.evaluate(X_valid, Y_valid, verbose = 2)
print("Evaluation, MAE: {:2.4f}, MSE: {:2.4f}".format(mae, mse))
elif sys.argv[1] == "predict":
# Step 3: Loads the weights
model.load_weights(model_name)
my_model = tf.keras.Sequential([model])
# Step 4: Get Normalization values
stats = pd.read_csv("data/wine_stats.csv", sep = ',', header = 0)
# Step 5: Prepare the input AND predict
input = sys.argv[2].split(",")
input = np.array([float(x) for x in input])
input = loader.normalize(input, stats).values
print(input)
def evaluate(self, dataset):
return evaluate(self.checkpoint.model, dataset)
# create required directory
required_dirs = ["param", "result", "mnist"]
hoge.make_dir(required_dirs)
mnist_data = MNIST('./mnist/',
train=True,
download=True,
transform=transforms.ToTensor())
dataloader = DataLoader(mnist_data, batch_size=mini_batch_num, shuffle=True)
print("\n")
# train restart
if interrupt_flag:
f = open("./param/tmp.pickle", mode="rb")
init_epoch = pickle.load(f)
model = model.GAN(dataloader, interrupting=True)
else:
init_epoch = 1
model = model.GAN(dataloader)
del dataloader
for epoch in range(init_epoch, epochs + 1):
print("Epoch[%d/%d]:" % (epoch, epochs))
model.study(epoch)
model.evaluate()
model.save_tmp_weight(epoch)
model.eval_pic(epoch)
model.output(epoch)
#model.output()
print('no model sess path specified. Can not preceed.')
sys.exit(1)
else:
result_path = cfg['sess_path']
valid_batch_data = get_batch_data(valid_text,
word2id_dict=word2id_dict,
batch_size=cfg['batch_size'],
max_length=cfg['max_length'])
pepfile_path = os.path.join(result_path, 'perplexity.txt')
result_ptr = open(pepfile_path, 'w')
print("start evaluate the language model")
evaluate(sess_path=result_path,
eva_data=valid_batch_data,
result_ptr=result_ptr)
print("evaluation phase completed")
if cfg['g']:
if not cfg['t']:
if len(cfg['sess_path']) == 0:
print('no model sess path specified. Can not preceed.')
sys.exit(1)
else:
result_path = cfg['sess_path']
# get id2word_dict
tf.reset_default_graph()
contin_path = "data/sentences.continuation"
contin_text = load_data(contin_path)
# Print best validation accuracy and epoch in valid_set
max_val_acc, idx = max((val, idx) for (idx, val) in enumerate(history.history['val_acc']))
print('Maximum accuracy at epoch', '{:d}'.format(idx + 1), '=', '{:.4f}'.format(max_val_acc))
# plot the result
import matplotlib.pyplot as plt
plt.figure()
plt.plot(history.epoch, history.history['acc'], label="acc")
plt.plot(history.epoch, history.history['val_acc'], label="val_acc")
plt.scatter(history.epoch, history.history['acc'], marker='*')
plt.scatter(history.epoch, history.history['val_acc'])
plt.legend(loc='lower right')
plt.show()
plt.figure()
plt.plot(history.epoch, history.history['loss'], label="loss")
plt.plot(history.epoch, history.history['val_loss'], label="val_loss")
plt.scatter(history.epoch, history.history['loss'], marker='*')
plt.scatter(history.epoch, history.history['val_loss'], marker='*')
plt.legend(loc='lower right')
plt.show()
score, acc = model.evaluate(X_test, test_label, batch_size=BATCH_SIZE)
print('Test score:', score)
print('Test accuracy:', acc)
predictions = model.predict(X_test)
preditFval(predictions, test_label)
import model
doc = {'dataset': {'path': 'data/sample1000/'},
# 'dataset': {'path': 'data/full/'},
'features': ['a', 'a1', 'a1-2', 'b', 'b1-3', 'd', 'd1'],
'fm_param': {'n_iter': 350, 'stdev': .001, 'rank': 4},
'seeds': [123, 345, 231, 500, 442, 873, 921, 111, 222],
'output': {},
'threshold': .84,
'shift_features': ['a', 'b', 'd'],
'max_shift': 5,
'min_freq': [1, 5, 10, 20],
'min_size': 10,
'lower_proba': .7,
'upper_proba': .9,
'submission_path': 'data/full/submission/submission.txt'
}
print model.evaluate(doc)
#model.submission(doc)
#try:
dxdt_guess=zeros(model.n_eq)
dxdt=zeros(model.n_eq)
x=zeros(model.n_eq)
for i in range(n_step):
It=I_at_time(time+dt)
flag=True
cnt=0
while True :
err=model.evaluate(dxdt,x,dxdt_guess,It,dt)
print cnt," Err=",err
cnt+=1
if err < tol or cnt>5:
break
dxdt[:]=dxdt_guess
x[:]=x+dt*dxdt
tt=copy.deepcopy(dxdt[model.n_node:model.n_node+n_plot_x])
plot.drawX(0,xx,tt)
times.append(time)
jj.append(tt)
def main(argv):
hvd.init()
validate_cmd_line_flags()
init_logging(log_path=FLAGS.log_path, FLAGS=FLAGS)
init_tf(FLAGS)
train_pipeline, validation_pipeline, dataset_metadata, multi_gpu_metadata = create_input_pipelines(FLAGS)
dlrm = Dlrm.load_model_if_path_exists(FLAGS.saved_model_input_path)
if dlrm is None:
if FLAGS.dummy_model:
dlrm = DummyDlrm(FLAGS=FLAGS, dataset_metadata=dataset_metadata,
multi_gpu_metadata=multi_gpu_metadata)
else:
dlrm = Dlrm(FLAGS=FLAGS, dataset_metadata=dataset_metadata,
multi_gpu_metadata=multi_gpu_metadata)
dlrm = dlrm.restore_checkpoint_if_path_exists(FLAGS.restore_checkpoint_path)
if FLAGS.optimizer == 'sgd':
embedding_optimizer = tf.keras.optimizers.SGD(learning_rate=FLAGS.learning_rate, momentum=0)
if FLAGS.amp:
embedding_optimizer = LossScaleOptimizer(embedding_optimizer,
initial_scale=FLAGS.loss_scale,
dynamic=False)
mlp_optimizer = embedding_optimizer
optimizers = [mlp_optimizer]
elif FLAGS.optimizer == 'adam':
embedding_optimizer = tfa.optimizers.LazyAdam(learning_rate=FLAGS.learning_rate)
mlp_optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate)
if FLAGS.amp:
embedding_optimizer = LossScaleOptimizer(embedding_optimizer,
initial_scale=FLAGS.loss_scale,
dynamic=False)
mlp_optimizer = LossScaleOptimizer(mlp_optimizer,
initial_scale=FLAGS.loss_scale,
dynamic=False)
optimizers = [mlp_optimizer, embedding_optimizer]
scheduler = LearningRateScheduler(optimizers,
warmup_steps=FLAGS.warmup_steps,
base_lr=FLAGS.learning_rate,
decay_start_step=FLAGS.decay_start_step,
decay_steps=FLAGS.decay_steps)
timer = IterTimer(train_batch_size=FLAGS.batch_size, test_batch_size=FLAGS.valid_batch_size,
optimizer=embedding_optimizer, print_freq=FLAGS.print_freq, enabled=hvd.rank() == 0)
splitter = DataParallelSplitter(batch_size=FLAGS.batch_size)
if FLAGS.mode == 'inference':
inference_benchmark(validation_pipeline, dlrm, timer, splitter, FLAGS)
return
elif FLAGS.mode == 'deploy':
dlrm.save_model_if_path_exists(FLAGS.saved_model_output_path,
save_input_signature=FLAGS.save_input_signature)
print('deployed to: ', FLAGS.saved_model_output_path)
return
elif FLAGS.mode == 'eval':
test_auc, test_loss, _ = evaluate(validation_pipeline, dlrm,
timer, auc_thresholds=FLAGS.auc_thresholds,
data_parallel_splitter=splitter)
if hvd.rank() == 0:
dllogger.log(data=dict(auc=test_auc, test_loss=test_loss), step=tuple())
return
eval_points = compute_eval_points(train_batches=len(train_pipeline),
evals_per_epoch=FLAGS.evals_per_epoch)
trainer = DlrmTrainer(dlrm, embedding_optimizer=embedding_optimizer,
mlp_optimizer=mlp_optimizer, amp=FLAGS.amp,
lr_scheduler=scheduler, dp_splitter=splitter,
data_parallel_bottom_mlp=FLAGS.data_parallel_bottom_mlp,
pipe=train_pipeline, cpu=FLAGS.cpu)
best_auc = 0
train_begin = time.time()
for epoch in range(FLAGS.epochs):
for step in range(len(train_pipeline)):
if step == FLAGS.profiler_start_step and hvd.rank() == FLAGS.profiled_rank:
tf.profiler.experimental.start('logdir')
if FLAGS.profiler_start_step and step == FLAGS.profiler_start_step + 100 and hvd.rank() == FLAGS.profiled_rank:
tf.profiler.experimental.stop()
loss = trainer.train_step()
timer.step_train(loss=loss)
if FLAGS.max_steps != -1 and step > FLAGS.max_steps:
dist_print(f'Max steps of {FLAGS.max_steps} reached, exiting')
break
if step in eval_points:
test_auc, test_loss, _ = evaluate(validation_pipeline, dlrm,
timer, FLAGS.auc_thresholds,
data_parallel_splitter=splitter)
dist_print(f'Evaluation completed, AUC: {test_auc:.6f}, test_loss: {test_loss:.6f}')
timer.test_idx = 0
best_auc = max(best_auc, test_auc)
elapsed = time.time() - train_begin
dlrm.save_checkpoint_if_path_exists(FLAGS.save_checkpoint_path)
dlrm.save_model_if_path_exists(FLAGS.saved_model_output_path,
save_input_signature=FLAGS.save_input_signature)
if hvd.rank() == 0:
dist_print(f'Training run completed, elapsed: {elapsed:.0f} [s]')
results = {
'throughput': FLAGS.batch_size / timer.mean_train_time(),
'mean_step_time_ms': timer.mean_train_time() * 1000,
'auc': best_auc
}
dllogger.log(data=results, step=tuple())
def main(args):
model_id = build_model_id(args)
model_path = build_model_path(args, model_id)
setup_model_dir(args, model_path)
sys.stdout, sys.stderr = setup_logging(args, model_path)
x_train, y_train = load_model_data(args.train_file,
args.data_name, args.target_name)
x_validation, y_validation = load_model_data(
args.validation_file,
args.data_name, args.target_name)
rng = np.random.RandomState(args.seed)
if args.n_classes > -1:
n_classes = args.n_classes
else:
n_classes = max(y_train)+1
n_classes, target_names, class_weight = load_target_data(args, n_classes)
if len(class_weight) == 0:
n_samples = len(y_train)
print('n_samples', n_samples)
print('classes', range(n_classes))
print('weights', n_samples / (n_classes * np.bincount(y_train)))
class_weight = dict(zip(range(n_classes),
n_samples / (n_classes * np.bincount(y_train))))
print('class_weight', class_weight)
logging.debug("n_classes {0} min {1} max {2}".format(
n_classes, min(y_train), max(y_train)))
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
min_vocab_index = np.min(x_train)
max_vocab_index = np.max(x_train)
logging.debug("min vocab index {0} max vocab index {1}".format(
min_vocab_index, max_vocab_index))
json_cfg = load_model_json(args, x_train, n_classes)
logging.debug("loading model")
sys.path.append(args.model_dir)
import model
from model import build_model
#######################################################################
# Subsetting
#######################################################################
if args.subsetting_function:
subsetter = getattr(model, args.subsetting_function)
else:
subsetter = None
def take_subset(subsetter, path, x, y, y_one_hot, n):
if subsetter is None:
return x[0:n], y[0:n], y_one_hot[0:n]
else:
mask = subsetter(path)
idx = np.where(mask)[0]
idx = idx[0:n]
return x[idx], y[idx], y_one_hot[idx]
x_train, y_train, y_train_one_hot = take_subset(
subsetter, args.train_file,
x_train, y_train, y_train_one_hot,
n=args.n_train)
x_validation, y_validation, y_validation_one_hot = take_subset(
subsetter, args.validation_file,
x_validation, y_validation, y_validation_one_hot,
n=args.n_validation)
#######################################################################
# Preprocessing
#######################################################################
if args.preprocessing_class:
preprocessor = getattr(model, args.preprocessing_class)(seed=args.seed)
else:
preprocessor = modeling.preprocess.NullPreprocessor()
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
model_cfg = ModelConfig(**json_cfg)
logging.info("model_cfg " + str(model_cfg))
model = build_model(model_cfg)
setattr(model, 'stop_training', False)
logging.info('model has {n_params} parameters'.format(
n_params=count_parameters(model)))
if len(args.extra_train_file) > 1:
callbacks = keras.callbacks.CallbackList()
else:
callbacks = []
save_model_info(args, model_path, model_cfg)
if not args.no_save:
if args.save_all_checkpoints:
filepath = model_path + '/model-{epoch:04d}.h5'
else:
filepath = model_path + '/model.h5'
callbacks.append(ModelCheckpoint(
filepath=filepath,
verbose=1,
save_best_only=not args.save_every_epoch))
callback_logger = logging.info if args.log else callable_print
if args.n_epochs < sys.maxsize:
# Number of epochs overrides patience. If the number of epochs
# is specified on the command line, the model is trained for
# exactly that number; otherwise, the model is trained with
# early stopping using the patience specified in the model
# configuration.
callbacks.append(EarlyStopping(
monitor='val_loss', patience=model_cfg.patience, verbose=1))
if args.classification_report:
cr = ClassificationReport(x_validation, y_validation,
callback_logger,
target_names=target_names)
callbacks.append(cr)
if model_cfg.optimizer == 'SGD':
callbacks.append(SingleStepLearningRateSchedule(patience=10))
if len(args.extra_train_file) > 1:
args.extra_train_file.append(args.train_file)
logging.info("Using the following files for training: " +
','.join(args.extra_train_file))
train_file_iter = itertools.cycle(args.extra_train_file)
current_train = args.train_file
callbacks._set_model(model)
callbacks.on_train_begin(logs={})
epoch = batch = 0
while True:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
iteration = batch % len(args.extra_train_file)
logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
epoch=epoch, iteration=iteration, train_file=current_train))
callbacks.on_epoch_begin(epoch, logs={})
n_train = x_train.shape[0]
callbacks.on_batch_begin(batch, logs={'size': n_train})
index_array = np.arange(n_train)
if args.shuffle:
rng.shuffle(index_array)
batches = keras.models.make_batches(n_train, model_cfg.batch_size)
logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
avg_train_loss = avg_train_accuracy = 0.
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(model, keras.models.Graph):
data = {
'input': x_train[batch_ids],
'output': y_train_one_hot[batch_ids]
}
train_loss = model.train_on_batch(data, class_weight=class_weight)
train_accuracy = 0.
else:
train_loss, train_accuracy = model.train_on_batch(
x_train[batch_ids], y_train_one_hot[batch_ids],
accuracy=True, class_weight=class_weight)
batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}
avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)
callbacks.on_batch_end(batch,
logs={'loss': train_loss, 'accuracy': train_accuracy})
logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))
batch += 1
# Validation frequency (this if-block) doesn't necessarily
# occur in the same iteration as beginning of an epoch
# (next if-block), so model.evaluate appears twice here.
kwargs = { 'verbose': 0 if args.log else 1 }
pargs = []
validation_data = {}
if isinstance(model, keras.models.Graph):
validation_data = {
'input': x_validation,
'output': y_validation_one_hot
}
pargs = [validation_data]
else:
pargs = [x_validation, y_validation_one_hot]
kwargs['show_accuracy'] = True
if (iteration + 1) % args.validation_freq == 0:
if isinstance(model, keras.models.Graph):
val_loss = model.evaluate(*pargs, **kwargs)
y_hat = model.predict(validation_data)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = model.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
callbacks.on_epoch_end(epoch, epoch_end_logs)
if batch % len(args.extra_train_file) == 0:
if isinstance(model, keras.models.Graph):
val_loss = model.evaluate(*pargs, **kwargs)
y_hat = model.predict(validation_data)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = model.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
epoch += 1
callbacks.on_epoch_end(epoch, epoch_end_logs)
if model.stop_training:
logging.info("epoch {epoch} iteration {iteration} - done training".format(
epoch=epoch, iteration=iteration))
break
current_train = next(train_file_iter)
x_train, y_train = load_model_data(current_train,
args.data_name, args.target_name)
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
if epoch > args.n_epochs:
break
callbacks.on_train_end(logs={})
else:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
if isinstance(model, keras.models.Graph):
data = {
'input': x_train,
'output': y_train_one_hot
}
validation_data = {
'input': x_validation,
'output': y_validation_one_hot
}
model.fit(data,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
validation_data=validation_data,
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
y_hat = model.predict(validation_data)
print('val_acc %.04f' %
accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1)))
else:
model.fit(x_train, y_train_one_hot,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
show_accuracy=True,
validation_data=(x_validation, y_validation_one_hot),
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
