def train_cnn(self,gpu_id=0,epoch_size=100,max_epoch=15,batch_size=128):
train_x, test_x, train_y, test_y = train_test_split(self.id_texts, self.labels_multi, random_state=0)
train = datasets.TupleDataset(train_x, train_y)
test = datasets.TupleDataset(test_x, test_y)
train_iter = iterators.SerialIterator(train, batch_size)
test_iter = iterators.SerialIterator(test, batch_size, False, False)
models = model.CNN(self.vocab_size, self.n_class)
models.embed.W.copydata(Variable(self.embedding_vectors))
if gpu_id >= 0:
models.to_gpu(gpu_id)
models = L.Classifier(models, lossfun=self.lossfun_multi_cnn, accfun=self.auc_fun_cnn)
optimizer = optimizers.Adam(alpha=0.001)
optimizer.setup(models)
updater = training.updaters.StandardUpdater(train_iter, optimizer, device=gpu_id)
trainer = training.Trainer(updater, (epoch_size * max_epoch, 'iteration'), out='./appr-cnn')
trainer.extend(extensions.LogReport(trigger=(epoch_size, 'iteration')))
trainer.extend(extensions.snapshot(filename='snapshot_iteration-{.updater.iteration}'), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.snapshot_object(models.predictor, filename='models_iteration-{.updater.iteration}'), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.Evaluator(test_iter, models, device=gpu_id), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.observe_lr(), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.PrintReport(['iteration', 'main/loss', 'validation/main/loss','main/accuracy', 'validation/main/accuracy', 'elapsed_time']), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.ProgressBar(update_interval=1000))
trainer.run()
def eval_cnn(self):
models = model.CNN(len(word2id), 6)
models_name = glob.glob("./appr-cnn/models_*")[-1]
serializers.load_npz(models_name, models)
gpu_id = 0
if gpu_id >= 0:
models.to_gpu(gpu_id)
models = L.Classifier(models, lossfun=lossfun_multi_cnn, accfun=auc_fun_cnn)
models.to_cpu()
div = len(test)//3
count = test_x.shape[0]//div
Bscore_list = []
auc_list = []
for i in range(count):
print(i)
pred_y = models.predictor(np.array(test_x[i*div:(i+1)*div]))
true_y = test_y[i*div:(i+1)*div]
Bscore_list.append(Bscore(true_y,pred_y))
auc_list.append(aucScore(true_y,pred_y))
pred_y = 0
true_y = 0
Bscore_list = np.array(Bscore_list)
auc_list = np.array(auc_list)
B_ave = []
auc_ave = []
for i in range(6):
B_ave.append(sum(Bscore_list[:,i])/count)
auc_ave.append(sum(auc_list[:,i])/count)
print("-"*50)
print(B_ave)
print(auc_ave)
def train_fn():
cnn_model = model.CNN()
# print(model.summary())
cnn_model.compile(optimizer=RMSprop(lr=config.LEARNING_RATE),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
train_images, train_labels, test_images, test_labels = data_preprocessing.data_augmentation()
earlystop = EarlyStopping(patience=10)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
verbose=1,
factor=0.5,
min_lr=0.00001)
csv_logger = CSVLogger(f'{config.MODEL_PATH}training.log', separator=',', append=False)
callbacks = [earlystop, learning_rate_reduction, csv_logger]
history = cnn_model.fit(train_images, train_labels,
epochs=config.NUM_EPOCHS,
verbose=2,
callbacks=callbacks)
print(cnn_model.evaluate(test_images, test_labels))
cnn_model.save(f"{config.MODEL_PATH}my_model.h5")
np.save(f'{config.MODEL_PATH}my_history.npy', history.history)
def train_fn():
cnn_model = model.CNN()
# print(model.summary())
cnn_model.compile(optimizer=RMSprop(lr=config.LEARNING_RATE),
loss='binary_crossentropy',
metrics=['accuracy'])
train_generator, valid_generator = data_preprocess.data_augmentation()
earlystop = EarlyStopping(patience=10)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
verbose=1,
factor=0.5,
min_lr=0.00001)
csv_logger = CSVLogger(f'{config.MODEL_PATH}training.log',
separator=',',
append=False)
callbacks = [earlystop, learning_rate_reduction, csv_logger]
history = cnn_model.fit(
train_generator,
validation_data=valid_generator,
steps_per_epoch=len(train_generator) // config.BATCH_SIZE,
validation_steps=len(valid_generator) // config.BATCH_SIZE,
epochs=config.NUM_EPOCHS,
verbose=2,
callbacks=callbacks)
cnn_model.save(f"{config.MODEL_PATH}my_model.h5")
np.save(f'{config.MODEL_PATH}my_history.npy', history.history)
def cross_validation(args):
for iter in range(1, args.num_models + 1):
args.lr = 10**np.random.uniform(-5, -1)
args.weight_decay = 10**np.random.uniform(-5, 1)
if args.modeltype in ['LR', 'MLP']:
args.factor = np.random.randint(90, 111)
model = models.LR(args) if args.modeltype == 'LR'\
else models.MLP(args)
print('{}, Model: {}, lr: {:.5f}, wd: {:.5f}, factor: {}'.format(
iter, args.modeltype, args.lr, args.weight_decay, args.factor))
f = open(args.filepath, 'a')
f.write('%d, Model: %s, lr: %.5f, wd: %.5f, factor: %d\n' %
(iter, args.modeltype, args.lr, args.weight_decay,
args.factor))
f.close()
elif args.modeltype in ['CNN', 'CNNDeep']:
args.embed_dim = np.random.randint(90, 111)
args.kernel_num = np.random.randint(90, 111)
args.dropout = np.random.uniform(0.1, 0.5)
model = models.CNN(args) if args.modeltype == 'CNN'\
else models.CNNDeep(args)
print('{}, Model: {}, lr: {:.5f}, wd: {:.5f}, embed_dim: {},\
kernel_num: {}, dropout: {:.5f}'.format(
iter, args.modeltype, args.lr, args.weight_decay,
args.embed_dim, args.kernel_num, args.dropout))
f = open(args.filepath, 'a')
f.write('%d, Model: %s, lr: %.5f, wd: %.5f, embed_dim: %d,\
kernel_num: %d, dropout: %.5f\n' %
(iter, args.modeltype, args.lr, args.weight_decay,
args.embed_dim, args.kernel_num, args.dropout))
f.close()
train_and_evaluate(args, model)
def main():
dims = ['cAGR', 'cCON', 'cEXT', 'cNEU', 'cOPN']
choice = int(sys.argv[1])
# preprocess set up
preprocessObj = Preprocessing()
paramsObj = config.Params()
preprocessObj.load_data(dims[choice])
preprocessObj.load_fasttext(paramsObj.use_word_embedding)
# set seed and cross validation
seed = 7
numpy.random.seed(seed)
kfolds = preprocessObj.cross_validation(preprocessObj.X, preprocessObj.Y,
seed)
cv_acc = []
cv_records = []
count_iter = 1
# loop the kfolds
for train, test in kfolds:
# create objects for each fold of 10-fold CV
modelObj = models.CNN()
# build the model
model = modelObj.build_simple_CNN(
paramsObj=paramsObj, weight=preprocessObj.embedding_matrix)
print(model.summary())
# save the best model & history
# filepath="weights.best.hdf5"
# checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
history = History()
callbacks_list = [history]
# fit the model
model.fit(preprocessObj.X[train],
preprocessObj.Y[train],
validation_data=(preprocessObj.X[test],
preprocessObj.Y[test]),
epochs=paramsObj.n_epoch,
batch_size=paramsObj.batch_size,
verbose=2,
callbacks=callbacks_list)
# record
ct = Counter(preprocessObj.Y)
print("----%s: %d----" % (preprocessObj.attribute, count_iter))
print("----highest evaluation accuracy is %f" %
(100 * max(history.history['val_acc'])))
print("----dominant distribution in data is %f" %
max([ct[k] * 100 / float(preprocessObj.Y.shape[0]) for k in ct]))
cv_acc.append(max(history.history['val_acc']))
cv_records.append(history.history['val_acc'])
count_iter += 1
print("The 10-fold CV score is %s" % np.nanmean(cv_acc))
def run():
#load config params to decide any pre-run actions needed
prerun_check()
cnn = model.CNN()
x, y = preprocess.load_train_images()
model.load_model(cnn)
model.train_model(cnn, x, y)
def main(args):
print ' Building the model'
yolo = model.CNN(args.alpha)
print 'Total parameters of the model are : ', yolo.count_params()
if args.t:
print 'Testing the model'
yolo.test(args.t)
def main():
data=torch.randn(4,109,64) #batch,channels,input for each channel
data=Variable(data)
model_final = model.CNN()
print(model_final)
output = model_final(data)
return output
def create_cam(config):
if not os.path.exists(config.result_path):
os.mkdir(config.result_path)
test_loader, num_class = utils.get_testloader(config.dataset,
config.dataset_path,
config.img_size)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cnn = model.CNN(img_size=config.img_size, num_class=num_class).to(device)
cnn.load_state_dict(
torch.load(os.path.join(config.model_path, config.model_name)))
finalconv_name = 'conv'
# hook
feature_blobs = []
def hook_feature(module, input, output):
feature_blobs.append(output.cpu().data.numpy())
cnn._modules.get(finalconv_name).register_forward_hook(hook_feature)
params = list(cnn.parameters())
# get weight only from the last layer(linear)
weight_softmax = np.squeeze(params[-2].cpu().data.numpy())
def returnCAM(feature_conv, weight_softmax, class_idx):
size_upsample = (config.img_size, config.img_size)
_, nc, h, w = feature_conv.shape
output_cam = []
cam = weight_softmax[class_idx].dot(feature_conv.reshape((nc, h * w)))
cam = cam.reshape(h, w)
cam = cam - np.min(cam)
cam_img = cam / np.max(cam)
cam_img = np.uint8(255 * cam_img)
output_cam.append(cv2.resize(cam_img, size_upsample))
return output_cam
for i, (image_tensor, label) in enumerate(test_loader):
image_PIL = transforms.ToPILImage()(image_tensor[0])
image_PIL.save(os.path.join(config.result_path, 'img%d.png' % (i + 1)))
image_tensor = image_tensor.to(device)
logit, _ = cnn(image_tensor)
h_x = F.softmax(logit, dim=1).data.squeeze()
probs, idx = h_x.sort(0, True)
print("True label : %d, Predicted label : %d, Probability : %.2f" %
(label.item(), idx[0].item(), probs[0].item()))
CAMs = returnCAM(feature_blobs[0], weight_softmax, [idx[0].item()])
img = cv2.imread(
os.path.join(config.result_path, 'img%d.png' % (i + 1)))
height, width, _ = img.shape
heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (width, height)),
cv2.COLORMAP_JET)
result = heatmap * 0.3 + img * 0.5
cv2.imwrite(os.path.join(config.result_path, 'cam%d.png' % (i + 1)),
result)
if i + 1 == config.num_result:
break
feature_blobs.clear()
def train(self, dataN):
classDatas = self.dataToClassFunc(dataN, self.thresholding)
data = dataset.CNN1DOnlineDataSet(raw_dataset=classDatas,
seq_len=self.timeSeriesLength)
self.model_lstm = model.CNN(self.config)
self.experiment = Experiment(config=self.config,
model=self.model_lstm,
dataset=data)
self.experiment.run()
def run(size, rank):
modell = model.CNN()
optimizer = torch.optim.Adam(modell.parameters(), lr=0.001)
loss_func = torch.nn.CrossEntropyLoss()
#size = torch.distributed.get_world_size()
#rank = torch.distributed.get_rank()
train_loader = Mnist().get_train_data()
test_data = Mnist().get_test_data()
test_x = torch.unsqueeze(test_data.test_data, dim=1).type(
torch.FloatTensor
)[:2000] / 255. # shape from (2000, 28, 28) to (2000, 1, 28, 28), value in range(0,1)
test_y = test_data.test_labels[:2000]
group_list = []
for i in range(size):
group_list.append(i)
group = torch.distributed.new_group(group_list)
print(rank, group_list)
for epoch in range(50):
for step, (b_x, b_y) in enumerate(train_loader):
modell = get_new_model(modell)
current_model = copy.deepcopy(modell)
output = modell(b_x)[0]
loss = loss_func(output, b_y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
new_model = copy.deepcopy(modell)
if step % 50 == 0:
test_output, last_layer = modell(test_x)
pred_y = torch.max(test_output, 1)[1].data.numpy()
accuracy = float(
(pred_y == test_y.data.numpy()).astype(int).sum()) / float(
test_y.size(0))
print('Epoch: ', epoch,
'| train loss: %.4f' % loss.data.numpy(),
'| test accuracy: %.2f' % accuracy)
for param1, param2 in zip(current_model.parameters(),
new_model.parameters()):
torch.distributed.reduce(param2.data - param1.data,
dst=0,
op=torch.distributed.reduce_op.SUM,
group=group)
def final_run(args):
if args.modeltype == 'LR':
model = models.LR(args)
elif args.modeltype == 'MLP':
model = models.MLP(args)
elif args.modeltype == 'CNN':
model = models.CNN(args)
elif args.modeltype == 'CNNDeep':
model = models.CNNDeep(args)
train_and_evaluate(args, model)
def master_run(size):
modell = model.CNN()
group = dist.new_group(range(size))
while True:
for param in modell.parameters():
dist.broadcast(param.data, src=0, group=group)
for param in modell.parameters():
tensor_temp = torch.zeros_like(param.data)
dist.reduce(tensor_temp, dst=0, op=dist.reduce_op.SUM, group=group)
param.data = tensor_temp / (size - 1)
def init_model(model_type):
model = []
if model_type == 'LeNet5':
model = fedmodel.LeNet5()
elif model_type == 'MLP':
model = fedmodel.MLP()
elif model_type == 'ResNet18':
model = fedmodel.ResNet18()
elif model_type == 'CNN':
model = fedmodel.CNN()
return model
def test(model_directory, batch_size):
# Import data
PIXEL_DEPTH = mnist_data.PIXEL_DEPTH
mnist = input_data.read_data_sets('data/', one_hot=True)
is_training = tf.placeholder(tf.bool, name='MODE')
# tf Graph input
x = tf.placeholder(tf.float32, [None, 784])
y_ = tf.placeholder(tf.float32, [None, 10]) # answer
y = model.CNN(x, is_training=is_training)
# Add ops to save and restore all the variables
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer(), feed_dict={is_training: True})
# Restore variables from disk
saver = tf.train.Saver()
# Calculate accuracy for all mnist test images
test_size = mnist.test.num_examples
total_batch = int(test_size / batch_size)
saver.restore(sess, model_directory)
acc_buffer = []
# Loop over all batches
for i in range(total_batch):
batch = mnist.test.next_batch(batch_size)
batch_xs = (
batch[0] - (PIXEL_DEPTH / 2.0) / PIXEL_DEPTH
) # make zero-centered distribution as in mnist_data.extract_data()
batch_ys = batch[1]
y_final = sess.run(y,
feed_dict={
x: batch_xs,
y_: batch_ys,
is_training: False
})
truth_labels = numpy.argmax(batch_ys, 1)
predict_labels = numpy.argmax(y_final, 1)
print("Truth: ", truth_labels)
print("Predict: ", predict_labels)
correct_prediction = numpy.equal(predict_labels, truth_labels)
acc_buffer.append(numpy.sum(correct_prediction) / batch_size)
print("test accuracy for the stored model: %g" % numpy.mean(acc_buffer))
def main():
print("\nParameters:")
for attr, value in args.__dict__.items():
print("\t{}={}".format(attr.upper(), value))
# load data
strain_data, sd_train_data, sdev_data, stest_data, embeddings =\
data_utils.load_dataset(args, 'askubuntu-master', dtrain=True)
dtrain_data, ddev_data, dtest_data, _ =\
data_utils.load_dataset(args, 'Android-master')
# initalize necessary parameters
args.embed_num = embeddings.shape[0]
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
# load model
if args.snapshot is None:
# initalize model
task_model = None
if args.model == 'lstm':
if args.bidirectional and (args.hidden_layer > 1):
args.hidden_layer = 1
print('\nMultilayer bidirectional LSTM not supported yet,\
layer set to 1.\n')
task_model = model.LSTM(args, embeddings)
elif args.model == 'cnn':
task_model = model.CNN(args, embeddings)
domain_model = model.DomainClassifier(args, embeddings)
# train models
res = train2.train_model(strain_data, sd_train_data, sdev_data,
stest_data, dtrain_data, ddev_data,
dtest_data, task_model, domain_model, args)
else:
print('\nLoading model from [%s]...' % args.snapshot)
try:
mod = torch.load(args.snapshot)
except:
print("Sorry, This snapshot doesn't exist.")
exit()
print(mod)
# evaluate
print('\nEvaluating on target dev')
evaluate.q_evaluate(mod, ddev_data, args)
print('Evaluating on target test')
evaluate.q_evaluate(mod, dtest_data, args)
def main(args):
print ' Building the model'
yolo = model.CNN(args)
print '\nTotal parameters of the model are : ', yolo.count_params()
if args.t:
print '\nTesting the model on test image(s)...'
yolo.test(args.t)
print '\nDone testing the model...'
if args.T:
print '\nBeginning training...'
yolo.train()
print '\nDone training the model...'
def content2sentence(table, model_path):
"""
Divide users' comment contents into separate sentence and
collect the postive or negtive scores
"""
sentiment_model = model.CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS,
FILTER_SIZES, OUTPUT_DIM, DROPOUT, PAD_IDX)
sentiment_model = sentiment_model.cuda()
sentiment_model.load_state_dict(torch.load(model_path))
nlp = spacy.load('en_core_web_lg')
TEXT = data.Field(tokenize='spacy',
batch_first=True,
tokenizer_language='en_core_web_lg')
LABEL = data.LabelField(dtype=torch.float)
vdata, _ = datasets.IMDB.splits(TEXT, LABEL, root='.data/')
TEXT.build_vocab(vdata,
max_size=25002,
vectors="glove.840B.300d",
vectors_cache='.vector_cache/',
unk_init=torch.Tensor.normal_)
with tqdm.tqdm(total=len(table), desc='Remove empty content') as pbar:
for i in range(len(table)):
if len(table.Content[i]) == 0:
table.drop(i, inplace=True)
pbar.update(1)
c2s = []
with tqdm.tqdm(total=len(table), desc='Processing...') as pbar:
for row in table.itertuples(index=False):
date = row.ReviewData
sentences = re.split('\!|\?|\.',
process_content(row.Content)[0])[:-1]
for sentence in sentences:
sentence = sentence.lower()
for c in string.punctuation:
sentence = sentence.replace(c, '')
if len(sentence.split(' ')) >= 10:
emotion_label = utils.predict_sentiment(sentiment_model,
sentence,
nlp,
TEXT,
min_len=5)
c2s.append([row.id, sentence, emotion_label, date])
pbar.update(1)
return pd.DataFrame(
c2s, columns=['cotent_id', 'sentence', 'emotion label', 'date'])
def main(_):
pprint.pprint(FLAGS.__flags)
# create directory
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
with tf.Session(config=run_config) as sess:
dcgan = model.CNN(sess,
input_width=FLAGS.input_width,
input_height=FLAGS.input_height,
batch_size=FLAGS.batch_size,
sample_num=FLAGS.batch_size,
y_dim=10,
dataset_name=FLAGS.dataset,
input_fname_pattern=FLAGS.input_fname_pattern,
crop=FLAGS.crop,
checkpoint_dir=FLAGS.checkpoint_dir,
sample_dir=FLAGS.sample_dir,
data_dir=FLAGS.data_dir)
show_all_variables()
if FLAGS.train:
dcgan.train(FLAGS)
else:
if not dcgan.load(FLAGS.checkpoint_dir)[0]:
raise Exception("[!] Train a model first, then run test mode")
# to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
# [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
# [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
# [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
# [dcgan.h4_w, dcgan.h4_b, None])
# Below is codes for visualization
OPTION = 1
visualize(sess, dcgan, FLAGS, OPTION)
def worker_run(size, rank):
modell = model.CNN()
optimizer = torch.optim.Adam(modell.parameters(), lr=LR)
loss_func = torch.nn.CrossEntropyLoss()
test_data = Mnist().get_test_data()
if (IID == True):
train_loader = Mnist().get_train_data()
else:
if (rank > 0):
if (rank == 1):
train_loader = Mnist_noniid().get_train_data1()
if (rank == 2):
train_loader = Mnist_noniid().get_train_data2()
if (rank == 3):
train_loader = Mnist_noniid().get_train_data3()
if (rank == 4):
train_loader = Mnist_noniid().get_train_data4()
if (rank == 5):
train_loader = Mnist_noniid().get_train_data5()
for step, (b_x, b_y) in enumerate(test_data):
test_x = b_x
test_y = b_y
group = dist.new_group(range(size))
for epoch in range(MAX_EPOCH):
modell = get_new_model(modell, group)
for step, (b_x, b_y) in enumerate(train_loader):
output = modell(b_x)[0]
loss = loss_func(output, b_y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
for param in modell.parameters():
dist.reduce(param.data, dst=0, op=dist.reduce_op.SUM, group=group)
test_output, last_layer = modell(test_x)
pred_y = torch.max(test_output, 1)[1].data.numpy()
accuracy = float(
(pred_y == test_y.data.numpy()).astype(int).sum()) / float(
test_y.size(0))
print('Epoch: ', epoch, ' Rank: ', rank,
'| train loss: %.4f' % loss.data.numpy(),
'| test accuracy: %.2f' % accuracy)
def train(config):
if not os.path.exists(config.model_path):
os.mkdir(config.model_path)
train_loader, num_class = utils.get_trainloader(config.dataset,
config.dataset_path,
config.img_size,
config.batch_size)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
cnn = model.CNN(img_size=config.img_size, num_class=num_class).to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(cnn.parameters(), lr=config.lr)
min_loss = 999
print("START TRAINING")
for epoch in range(config.epoch):
epoch_loss = 0
for i, (images, labels) in enumerate(train_loader):
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
outputs, _ = cnn(images)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
epoch_loss += loss.item()
if (i + 1) % config.log_step == 0:
if config.save_model_in_epoch:
torch.save(cnn.state_dict(), os.path.join(config.model_path, config.model_name))
print('Epoch [%d/%d], Iter [%d/%d], Loss: %.4f'
% (epoch + 1, config.epoch, i + 1, len(train_loader), loss.item()))
avg_epoch_loss = epoch_loss / len(train_loader)
print('Epoch [%d/%d], Loss: %.4f'
% (epoch + 1, config.epoch, avg_epoch_loss))
if avg_epoch_loss < min_loss:
min_loss = avg_epoch_loss
torch.save(cnn.state_dict(), os.path.join(config.model_path, config.model_name))
def train_fn():
X, y = data_preprocess.data_preprocessing(config.TRAIN_DF)
X_test, y_test = data_preprocess.data_preprocessing(config.TEST_DF)
X_train, X_val, y_train, y_val = model_selection.train_test_split(
X, y, test_size=config.TEST_SIZE, random_state=100, stratify=y)
cnn_model = model.CNN()
cnn_model.compile(optimizer=RMSprop(lr=config.LEARNING_RATE),
loss='categorical_crossentropy',
metrics=['accuracy'])
# print(model.summary())
earlystop = EarlyStopping(patience=10)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
verbose=1,
factor=0.5,
min_lr=0.00001)
csv_logger = CSVLogger(f'{config.MODEL_PATH}training.log',
separator=',',
append=False)
callbacks = [earlystop, learning_rate_reduction, csv_logger]
history = cnn_model.fit(X_train,
y_train,
validation_data=(X_val, y_val),
epochs=config.NUM_EPOCHS,
verbose=2,
callbacks=callbacks)
score = cnn_model.evaluate(X_test, y_test, verbose=0)
print(score)
cnn_model.save(f"{config.MODEL_PATH}my_model.h5")
np.save(f'{config.MODEL_PATH}my_history.npy', history.history)
def main(game, play_against, to_solve, model_file):
global new_model, original, game_type
new_model = model.CNN(game).cuda()
original = copy.deepcopy(new_model)
game_type = game
if play_against or to_solve:
with open(model_file, 'rb') as f:
new_model = torch.load(f)
if to_solve:
solve()
else:
print('Score against {}: {}/{}.'.format(model_file,
play(True, True),
config.EVAL_ITERS))
else:
for i in range(config.FULL_ITERS):
print('Starting full iteration #{}...'.format(i + 1))
play(True, False)
train()
evaluate(i + 1)
final_test()
def run():
modell = model.CNN()
# modell = model.AlexNet()
size = dist.get_world_size()
rank = dist.get_rank()
group_list = []
for i in range(size):
group_list.append(i)
group = dist.new_group(group_list)
while (1):
for param in modell.parameters():
# for dst in range(1, size):
# dist.send(param.data, dst=dst)
dist.broadcast(param.data, src=0, group=group)
for param in modell.parameters():
tensor_temp = torch.zeros_like(param.data)
dist.reduce(tensor_temp, dst=0, op=dist.reduce_op.SUM, group=group)
param.data = tensor_temp / (size - 1)
def run():
modell = model.CNN()
size = torch.distributed.get_world_size()
rank = torch.distributed.get_rank()
group_list = []
for i in range(size):
group_list.append(i)
group = torch.distributed.new_group(group_list)
while(1):
for param in modell.parameters():
torch.distributed.broadcast(param.data, src=0, group=group)
#param_list = []
for param in modell.parameters():
#param_list.append(param.data)
tensor_update = torch.zeros_like(param.data)
torch.distributed.reduce(tensor_update, dst=0, op=torch.distributed.reduce_op.SUM, group=group)
tensor_update /= (size-1)
param.data += tensor_update
def main():
print("\nParameters:")
for attr, value in args.__dict__.items():
print("\t{}={}".format(attr.upper(), value))
# load data
train_data, embeddings = data_utils.load_dataset()
# initalize necessary parameters
args.embed_num = embeddings.shape[0]
args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')]
# choose model
args.model = 'lstm'
# args.model = 'cnn'
# load model
if args.model == 'lstm':
mod = model.LSTM(args, embeddings)
else:
mod = model.CNN(args, embeddings)
# train model
res = train.train_model(train_data, mod, args)
devices_list.update({'gpu{}'.format(i): i for i in range(1, args.n_gpus)})
print(devices_list)
updater = training.updaters.ParallelUpdater(train_iter, optimizer, devices=devices_list)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
evaluator = extensions.Evaluator(valid_iter, model, device=master_gpu_id)
# Set some extension modules to a trainer.
prepare_extensions(trainer, evaluator, args)
# Run the training
trainer.run()
# Show real throughput.
datasize = len(train) * args.epoch
throughput = datasize / trainer.elapsed_time
print('Throughput: {} [images/sec.] ({} / {})'.format(
throughput, datasize, trainer.elapsed_time))
# Save trained model.
model_filepath = os.path.join(args.out, 'trained.model')
chainer.serializers.save_npz(model_filepath, model)
if __name__ == '__main__':
args = parse_cmd_args()
with open("train.pkl", "rb") as f:
x, t = pickle.load(f)
M = L.Classifier(model.CNN())
if args.n_gpus > 0:
train_using_gpu(args, M, x, t)
import os
import model
import tensorflow as tf
import data_utils
import configuration
data, labels, w2idx = data_utils.get_data(configuration.config['dataset'])
configuration.config['n_words'] = len(w2idx) + 1
with tf.Session() as sess:
net = model.CNN(configuration.config, sess, w2idx)
net.train(data, labels)
def run(size, rank):
modell = model.CNN()
#modell = model.AlexNet()
optimizer = torch.optim.Adam(modell.parameters(), lr=LR)
loss_func = torch.nn.CrossEntropyLoss()
if(IID == True):
train_loader = Mnist().get_train_data()
test_data = Mnist().get_test_data()
else:
if(rank > 0):
if(rank == 1):
train_loader = Mnist_noniid().get_train_data1()
test_data = Mnist_noniid().get_test_data1()
if(rank == 2):
train_loader = Mnist_noniid().get_train_data2()
test_data = Mnist_noniid().get_test_data2()
if(rank == 3):
train_loader = Mnist_noniid().get_train_data3()
test_data = Mnist_noniid().get_test_data3()
if(rank == 4):
train_loader = Mnist_noniid().get_train_data4()
test_data = Mnist_noniid().get_test_data4()
if(rank == 5):
train_loader = Mnist_noniid().get_train_data5()
test_data = Mnist_noniid().get_test_data5()
#size = dist.get_world_size()
#rank = dist.get_rank()
#train_loader = Mnist().get_train_data()
#test_data = Mnist().get_test_data()
for step, (b_x, b_y) in enumerate(test_data):
test_x = b_x
test_y = b_y
group_list = []
for i in range(size):
group_list.append(i)
group = dist.new_group(group_list)
for epoch in range(MAX_EPOCH):
modell = get_new_model(modell, group)
#current_model = copy.deepcopy(modell)
test_output, last_layer = modell(test_x)
pred_y = torch.max(test_output, 1)[1].data.numpy()
accuracy = float((pred_y == test_y.data.numpy()).astype(int).sum()) / float(test_y.size(0))
for step, (b_x, b_y) in enumerate(train_loader):
#modell = get_new_model(modell)
#current_model = copy.deepcopy(modell)
output = modell(b_x)[0]
loss = loss_func(output, b_y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
for param in modell.parameters():
dist.reduce(param.data, dst=0, op=dist.reduce_op.SUM, group=group)
f = open('./test.txt', 'a')
print('Epoch: ', epoch, ' Rank: ', rank, '| train loss: %.4f' % loss.data.numpy(),
'| test accuracy: %.2f' % accuracy, file=f)
print('Epoch: ', epoch, ' Rank: ', rank, '| train loss: %.4f' % loss.data.numpy(),
'| test accuracy: %.2f' % accuracy)
f.close()
