def main(args):
preprocessor = Preprocessor(FLAGS)
word_dict = preprocessor.build_dict()
embeddings = preprocessor.get_init_embedding()
model = Model(embeddings, FLAGS, len(word_dict))
model.build()
if FLAGS.mode == "train":
train_model(model, preprocessor, word_dict, FLAGS)
elif FLAGS.mode == "test":
test_model(model, preprocessor, word_dict, FLAGS)
def post(self):
# Read config file
model_name = config["model_name"]
img_width = config["img_width"]
img_height = config["img_height"]
classes = config["classes"]
args = file_upload.parse_args()
args['img'].save(
os.path.join('test_images/',
secure_filename(args['img'].filename)))
# Load trained model
model_path = 'models/{}'.format(model_name)
model = load_model(model_path)
# Attributes
img_path = os.path.join('test_images/',
secure_filename(args['img'].filename))
predicted_classes = test_model(img_path, classes, model, img_width,
img_height)
return {'result': predicted_classes}
def train_model(para, data, path_excel):
## data and hyperparameters
[train_data, train_data_interaction, user_num, item_num, test_data, pre_train_feature, hypergraph_embeddings, graph_embeddings, propagation_embeddings, sparse_propagation_matrix, _] = data
[_, _, MODEL, LR, LAMDA, LAYER, EMB_DIM, BATCH_SIZE, TEST_USER_BATCH, N_EPOCH, IF_PRETRAIN, _, TOP_K] = para[0:13]
if MODEL == 'LightLCFN': [_, _, _, KEEP_PORB, SAMPLE_RATE, GRAPH_CONV, PREDICTION, LOSS_FUNCTION, GENERALIZATION, OPTIMIZATION, IF_TRASFORMATION, ACTIVATION, POOLING] = para[13:]
if MODEL == 'SGNN': [_, PROP_EMB, _] = para[13:]
para_test = [train_data, test_data, user_num, item_num, TOP_K, TEST_USER_BATCH]
## Define the model
if MODEL == 'MF': model = model_MF(n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA)
if MODEL == 'NCF': model = model_NCF(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN)
if MODEL == 'GCMC': model = model_GCMC(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, sparse_graph=sparse_propagation_matrix)
if MODEL == 'NGCF': model = model_NGCF(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, sparse_graph=sparse_propagation_matrix)
if MODEL == 'SCF': model = model_SCF(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, sparse_graph=sparse_propagation_matrix)
if MODEL == 'CGMC': model = model_CGMC(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, sparse_graph=sparse_propagation_matrix)
if MODEL == 'LightGCN': model = model_LightGCN(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, sparse_graph=sparse_propagation_matrix)
if MODEL == 'LCFN': model = model_LCFN(layer=LAYER, n_users=user_num, n_items=item_num, emb_dim=EMB_DIM, lr=LR, lamda=LAMDA, pre_train_latent_factor=pre_train_feature, if_pretrain=IF_PRETRAIN, graph_embeddings=hypergraph_embeddings)
if MODEL == 'LightLCFN': model = model_LightLCFN(n_users=user_num, n_items=item_num, lr=LR, lamda=LAMDA, emb_dim=EMB_DIM, layer=LAYER, pre_train_latent_factor=pre_train_feature, graph_embeddings=graph_embeddings, graph_conv = GRAPH_CONV, prediction = PREDICTION, loss_function=LOSS_FUNCTION, generalization = GENERALIZATION, optimization=OPTIMIZATION, if_pretrain=IF_PRETRAIN, if_transformation=IF_TRASFORMATION, activation=ACTIVATION, pooling=POOLING)
if MODEL == 'SGNN': model = model_SGNN(n_users=user_num, n_items=item_num, lr=LR, lamda=LAMDA, emb_dim=EMB_DIM, layer=LAYER, pre_train_latent_factor=pre_train_feature, propagation_embeddings=propagation_embeddings, if_pretrain=IF_PRETRAIN, prop_emb=PROP_EMB)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
## Split the training samples into batches
batches = list(range(0, len(train_data_interaction), BATCH_SIZE))
batches.append(len(train_data_interaction))
## Training iteratively
F1_max = 0
F1_df = pd.DataFrame(columns=TOP_K)
NDCG_df = pd.DataFrame(columns=TOP_K)
t1 = time.clock()
for epoch in range(N_EPOCH):
for batch_num in range(len(batches) - 1):
train_batch_data = []
for sample in range(batches[batch_num], batches[batch_num + 1]):
(user, pos_item) = train_data_interaction[sample]
sample_num = 0
while sample_num < (SAMPLE_RATE if MODEL == 'LightLCFN' else 1):
neg_item = int(rd.uniform(0, item_num))
if not (neg_item in train_data[user]):
sample_num += 1
train_batch_data.append([user, pos_item, neg_item])
train_batch_data = np.array(train_batch_data)
_, loss = sess.run([model.updates, model.loss], feed_dict={model.users: train_batch_data[:, 0], model.pos_items: train_batch_data[:, 1], model.neg_items: train_batch_data[:, 2], model.keep_prob: KEEP_PORB if MODEL == 'LightLCFN' else 1})
## test the model each epoch
F1, NDCG = test_model(sess, model, para_test)
F1_max = max(F1_max, F1[0])
## print performance
# print_value([epoch + 1, loss, F1_max, F1, NDCG])
if epoch % 10 == 0: print('%.5f' % (F1_max), end = ' ', flush = True)
## save performance
F1_df.loc[epoch + 1] = F1
NDCG_df.loc[epoch + 1] = NDCG
save_value([[F1_df, 'F1'], [NDCG_df, 'NDCG']], path_excel, first_sheet=False)
if loss > 10 ** 10: break
t2 = time.clock()
print('time cost:', (t2 - t1) / 200)
return F1_max
def imitate(args):
# initialization
epoch = args.num_epochs
train_obs, train_act = load_initial_data(args)
batch = args.batch_size
transform = args.transform
args.dataset_sizes = len(train_obs)
args.dataloader = load_dataset(args,train_obs,train_act,batch, transform)
num_valid_episode = args.num_valid_episode
logger = args.logger
final_positions = []
success_history = []
frames = []
reward_history = []
if args.do_dagger: epoch += args.max_dagger_iterations
for e in range(epoch):
# train
print('epoch: {}'.format(e+1))
args.model = train_model(args)
# dagger
if args.do_dagger and e>=args.num_epochs:
imit_obs, exp_act = execute_dagger(args)
train_obs, train_act = aggregate_dataset(train_obs, train_act,
imit_obs, exp_act)
args.dataset_sizes = len(train_obs)
args.dataloader = load_dataset(args,train_obs,train_act,batch, transform)
# valid
total_reward = 0
total_success = 0
for i_episode in range(num_valid_episode):
final_position, success, frames_ep, episode_reward \
= test_model(args)
total_success += success
total_reward += episode_reward
final_positions.append(final_position)
# success_history.append(success)
# reward_history.append(episode_reward)
frames.append(frames_ep)
success_rate = total_success / num_valid_episode
mean_reward = total_reward / num_valid_episode
success_history.append(success_rate)
reward_history.append(mean_reward)
logger.add_scalar('success rate', success_rate, e+1)
logger.add_scalar('mean reward', mean_reward, e+1)
print('success rate: {} mean reward: {}'.format(success_rate,
mean_reward))
return final_positions, success_history, frames, reward_history, args
def cross_validation(structure_id, model_name, max_epochs, train_tweets,
train_labels, validate_tweets, validate_labels,
test_tweets):
train_tweetsA = []
train_labelsA = []
validate_tweetsA = []
validate_labelsA = []
for i in range(len(train_tweets)):
if train_labels[i] == 0:
train_tweetsA.append(train_tweets[i])
train_labelsA.append(train_labels[i])
elif train_labels[i] == 1:
train_tweetsA.append(train_tweets[i])
train_labelsA.append(train_labels[i])
else:
train_tweetsA.append(train_tweets[i])
train_labelsA.append(train_labels[i])
train_labelsA[i] = 1
for i in range(len(validate_tweets)):
if validate_labels[i] == 0:
validate_tweetsA.append(validate_tweets[i])
validate_labelsA.append(validate_labels[i])
elif validate_labels[i] == 1:
validate_tweetsA.append(validate_tweets[i])
validate_labelsA.append(validate_labels[i])
else:
validate_tweetsA.append(validate_tweets[i])
validate_labelsA.append(validate_labels[i])
validate_labelsA[i] = 1
train_tweetsB = []
train_labelsB = []
validate_tweetsB = []
validate_labelsB = []
for i in range(len(train_tweets)):
if train_labels[i] == 1:
train_tweetsB.append(train_tweets[i])
train_labelsB.append(train_labels[i] - 1)
elif train_labels[i] == 2:
train_tweetsB.append(train_tweets[i])
train_labelsB.append(train_labels[i] - 1)
for i in range(len(validate_tweets)):
if validate_labels[i] == 1:
validate_tweetsB.append(validate_tweets[i])
validate_labelsB.append(validate_labels[i] - 1)
elif validate_labels[i] == 2:
validate_tweetsB.append(validate_tweets[i])
validate_labelsB.append(validate_labels[i] - 1)
create_model(0, model_name, classes, max_epochs, train_tweets,
train_labels, validate_tweets, validate_labels)
return test_model(model_name, classes, subclasses, binary, test_tweets)
def clean_and_test(directory, model_file, classifierType, birds, verbose,
skip_clean, no_sanitize):
if not len(birds):
raise Exception("Must specify at least one folder/category to test!")
start_time = time.clock()
# birds = ['bluejay_all', 'cardinal_all', 'cardinal_call', 'cardinal_song', 'chickadee_all', 'chickadee_call',
# 'chickadee_song', 'crow_all', 'goldfinch_all', 'goldfinch_call', 'goldfinch_song', 'robin_all',
# 'robin_call', 'robin_song', 'sparrow_all', 'sparrow_call',
# 'sparrow_song', 'titmouse_all', 'titmouse_call', 'titmouse_song']
test_dirs = test_params(directory, birds)
try:
if not no_sanitize:
sanatize_filenames(directory, verbose=verbose)
if not skip_clean:
for dir in test_dirs:
rootdir, subdir = os.path.split(dir)
cleaner = noiseCleaner(verbose=verbose)
cleaner.noise_removal_dir(rootdir)
model_dir, model_name = os.path.split(model_file)
if not len(model_dir):
model_dir = os.getcwd()
test_model(test_dirs,
modelName=model_name,
model_dir=model_dir,
classifierType=classifierType,
store_to_mySQL=True,
verbose=verbose)
except Exception:
send_notification("Clean and test process failed.")
raise
else:
send_notification("Clean and test finished successfully.")
print "Total time elapsed: ", time.clock() - start_time, " seconds."
if node_classification:
testtrain_loader = DataLoader(train_dataset,
batch_size=1,
shuffle=False)
else:
testtrain_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=False)
model_filename = "topk_gat_test_notnull_notrandom.pt"
# model = GCNNet(dataset.num_features, dataset.num_classes)
model = TopKNet(train_dataset.num_features, train_dataset.num_classes)
model.load_state_dict(torch.load(model_filename))
# model = AGNNNet(dataset.num_features, dataset.num_classes)
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(),
lr=0.0001,
weight_decay=1e-4)
for epoch in range(190, 1001):
loss = train()
if epoch % 10 == 0:
train_accuracy = test_model.test_model(testtrain_loader, model,
node_classification, device)
test_accuracy = test_model.test_model(test_loader, model,
node_classification, device)
log = 'Epoch: {:03d}, Train Loss: {:.8f} Train Accuracy: {:.8f} Test Accuracy: {:.8f}'
print(log.format(epoch, loss, train_accuracy, test_accuracy))
print("Saving model topk")
torch.save(model.state_dict(), model_filename)
print('Model: ' + model_filename + ' is saved.')
def main():
'''
Creates and train a model for an image folder.
Uses a pretrained model.
If --resume_training_checkpoint argument is set, it will load a
model checkpoint and resume training it instead of creating a new model.
Usage:
python train.py data_directory [--save_dir] <saving_directory> [--saving_name] <saving_name>
[--arch] <pretrained_model_arch> [--learning_rate] <learning_rate> [--epochs] <training_epochs>
[--hidden_units1] <hidden_units_1st_hidden_layer> [--hidden_units2] <hidden_units_2nd_hidden_layer>
[--resume_training_checkpoint] <path_to_trained_model_checkpoint> [--gpu]
If --resume_training_checkpoint argument is set, the following arguments will be ignored:
--arch, --learning_rate, --hidden_units1, --hidden_units2
Defaults:
--checkpoint_name : "checkpoint.pth",
--arch : "vgg19",
--learning_rate : 0.001,
--epochs : 3,
--hidden_units1 : 512,
--hidden_units2 : 256,
--resume_training_checkpoint : None,
--gpu : False
'''
#get command line arguments
args = get_input_args()
#check device
try:
device = check_gpu(args.gpu)
except Exception as e:
print(e)
sys.exit(0)
#print selected arguments
print("Selected arguments:")
print("\tData Folder = {}".format(args.data_directory))
print("\tCheckpoint save Folder = {}".format(args.save_dir))
print("\tCheckpoint save Name = {}".format(args.saving_name))
print("\tEpochs = {}".format(args.epochs))
if args.resume_training_checkpoint:
print("\tTrained model checkpoint = {}".format(
args.resume_training_checkpoint))
else:
print("\tPretrained model architecture = {}".format(args.arch))
print("\tTraining learning rate = {}".format(args.learning_rate))
print("\tNumber of hidden units (1st hidden layer) = {}".format(
args.hidden_units1))
print("\tNumber of hidden units (2nd hidden layer) = {}".format(
args.hidden_units2))
print("\tSelected Device = {}".format(device))
#load data
train_data, validation_data, test_data = get_datasets(args.data_directory)
trainloader, validationloader, testloader = get_dataloaders(
train_data, validation_data, test_data)
flower_categories = 102
already_trained_epochs = 0
#creates or loads a model, depending of argument resume_training_checkpoint
if args.resume_training_checkpoint:
print("\nLoading model and resuming training!")
model, criterion, optimizer, already_trained_epochs = load_model_checkpoint(
args.resume_training_checkpoint)
#need to move optimizer tensors to device to keep training,
#if using an already trained checkpoint
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
if device == "cuda":
state[k] = v.cuda()
else:
state[k] = v.cpu()
else:
print("\nCreating new model!")
model, criterion, optimizer = create_model(args.arch,
args.hidden_units1,
args.hidden_units2,
flower_categories,
args.learning_rate)
model.to(device)
#trains and validates model
train_and_validate_model(device, model, criterion, optimizer, args.epochs,
already_trained_epochs, trainloader,
validationloader)
#tests model
test_model(device, model, testloader)
#saves model checkpoint
epochs = already_trained_epochs + args.epochs
save_checkpoint(args.arch, args.hidden_units1, args.hidden_units2,
flower_categories, train_data, epochs, args.learning_rate,
model, optimizer, args.save_dir, args.saving_name)
print("\nEnd of training script.")
def train_model(model,
loss_fn,
batch_size,
dataset,
optimizer,
model_title='None',
device='cpu',
root_dir='',
num_epochs=2,
testset=None):
def write_results(title, write_string):
with open('model_results/' + title + '_results.txt', 'a+') as f:
print(write_string)
f.write(write_string + '\n')
# Shuffling is needed in case dataset is not shuffled by default.
train_ratio = .9
train_len = int(len(dataset) * train_ratio)
train_dataset, val_dataset = torch.utils.data.random_split(
dataset, [train_len, len(dataset) - train_len])
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
collate_fn=my_collate)
# We don't need to bach the validation set but let's do it anyway.
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size,
collate_fn=my_collate)
# GPU enabling.
model = model.to(device)
loss_fn = loss_fn.to(device)
# Training loop. Please make sure you understand every single line of code below.
# Go back to some of the previous steps in this lab if necessary.
start_time = time.time()
plot_path = os.path.join(root_dir, 'model_results/' + model_title + '.png')
print("Trainset Length: ", len(train_loader))
print("Valset Length: ", len(val_loader))
print("Epoch Count: ", num_epochs)
print("Plot Path: ", plot_path)
train_accuracies = []
val_accuracies = []
train_losses = []
val_losses = []
for epoch in range(0, num_epochs):
correct = 0.0
cum_loss = 0.0
# Make a pass over the training data.
model.train()
num_trained = 1.0
confusion_matrix = torch.zeros(len(dataset.labels),
len(dataset.labels))
for (i, (inputs, labels)) in enumerate(train_loader):
try:
inputs = inputs.to(device)
labels = labels.to(device) #[0]
# Forward pass. (Prediction stage)
scores = model(inputs)
loss = loss_fn(scores, labels)
# Zero the gradients in the network.
optimizer.zero_grad()
# Backward pass. (Gradient computation stage)
loss.backward()
# Parameter updates (SGD step) -- if done with torch.optim!
optimizer.step()
scores = F.softmax(scores)
max_scores, max_labels = scores.max(1)
correct_eval = (max_labels == labels).sum().item()
correct += correct_eval
num_trained += batch_size
cum_loss += loss.item()
_, preds = torch.max(scores, 1)
for t, p in zip(labels.view(-1), preds.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
# Logging the current results on training.
if (i + 1) % 100 == 0:
print(dataset.labels)
write_string = (
'Train-epoch %d. Iteration %05d, Avg-Loss: %.4f, Accuracy: %.4f'
% (epoch, num_trained + 1, cum_loss /
(num_trained), correct / (num_trained)))
write_string += '\n' + str(confusion_matrix)
confusion_avg = (confusion_matrix.diag() /
confusion_matrix.sum(1))
write_string += '\n' + str(confusion_avg)
write_results(model_title, write_string)
#print("Time Elapsed Minutes: ", (time.time() - start_time) /60)
correct = 0.0
cum_loss = 0.0
num_trained = 1.0
confusion_matrix = torch.zeros(len(dataset.labels),
len(dataset.labels))
model_save_path = 'saved_models/' + model_title + '.pth'
torch.save(model.state_dict(), model_save_path)
except Exception as e:
print(e)
train_accuracies.append(correct / num_trained)
train_losses.append(cum_loss / num_trained)
write_string = "Time Elapsed Minutes: " + str(
(time.time() - start_time) / 60)
write_results(model_title, write_string)
# Make a pass over the validation data.
model.eval()
num_trained = 1.0
correct = 0.0
cum_loss = 0.0
confusion_matrix = torch.zeros(len(dataset.labels),
len(dataset.labels))
print('Validating...')
print(dataset.labels)
for (i, (inputs, labels)) in enumerate(val_loader):
try:
inputs = inputs.to(device)
labels = labels.to(device) #[0]
# Forward pass. (Prediction stage)
scores = model(inputs)
latent_scores.append([scores.item(), labels.item()])
loss = loss_fn(scores, labels)
scores = F.softmax(scores)
# Count how many correct in this batch.
max_scores, max_labels = scores.max(1)
correct_eval = (max_labels == labels).sum().item()
correct += correct_eval
num_trained += batch_size
cum_loss += loss.item()
_, preds = torch.max(scores, 1)
for t, p in zip(labels.view(-1), preds.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
if i % 100 == 0:
print(confusion_matrix)
except Exception as e:
print(e)
val_accuracies.append(correct / num_trained)
val_losses.append(cum_loss / num_trained)
write_string = (
'validation-epoch %d. Iteration %05d, Avg-Loss: %.4f, Accuracy: %.4f'
% (epoch, num_trained + 1, cum_loss / num_trained,
correct / num_trained))
write_string += '\n' + str(confusion_matrix)
confusion_avg = (confusion_matrix.diag() / confusion_matrix.sum(1))
write_string += '\n' + str(confusion_avg)
write_results(model_title, write_string)
plot_model_stats(train_accuracies,
val_accuracies,
train_losses,
val_losses,
plot_path,
latent_scores=latent_scores)
if testset != None:
try:
from test_model import test_model
test_model(model,
testset,
dataset,
batch_size,
joint_run=joint_run)
except Exception as e:
print(e)
def main():
train_model.train_model()
test_model.test_model()
def sample_interval(self, epoch, test_X, test_y, experiment):
test_model(self, epoch, test_X, test_y, self.experiment)
n_class=len(np.unique(np.argmax(y_train, 1))),
routings=args.routings)
model.summary()
# train or test
if args.weights is not None: # init the model weights with provided one
model.load_weights(args.weights)
if not args.testing:
train(model=model, data=((x_train, y_train), (x_test_all, y_test_all)), args=args)
else: # as long as weights are given, will run testing
if args.weights is None:
print('No weights are provided. Will test using random initialized weights.')
if not is_only_3_and_4:
test_model(eval_model, args.n_test, x_test_all, y_test_all,
BATCH_SIZE, ang_min, ang_max)
else:
nImg = x_test_all.shape[0]
x_test = np.empty([1, sy, sx, 1], dtype=np.float32)
y_test = np.empty([1, NCLASSES])
k = 0
for i in range(nImg):
y_i = y_test_all[i, :]
y_i = np.argmax(y_i)
if (y_i == 3) or (y_i == 4):
if k == 0:
x_test[0, :, :, :] = x_test_all[i, :, :, :]
y_test[0, :] = y_test_all[i, :]
else:
x_test = np.concatenate([x_test, np.expand_dims(x_test_all[i, :, :, :], 0)])
y_test = np.concatenate([y_test, np.expand_dims(y_test_all[i, :], 0)])
torch.manual_seed(0)
np.random.seed(0)
batch_size = 2
num_epochs = 5
num_classes = 5
learning_rate = 0.003
model = BiLSTM(batch_size=batch_size)
list_of_files = glob.glob(models_dir + '*.ckpt')
if list_of_files:
latest_file = max(list_of_files, key=os.path.getctime)
model.load_state_dict(torch.load(latest_file))
model.to(device)
ds = SequenceDataset()
vocab = ds.get_vocab()
train_len = int(len(ds) * 0.8)
test_len = len(ds) - train_len
train_ds, test_ds = torch.utils.data.random_split(ds, [train_len, test_len])
train_dl = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True)
test_dl = torch.utils.data.DataLoader(test_ds, batch_size=batch_size, shuffle=True)
model = train_model(model, vocab, train_dl, learning_rate, num_epochs)
test_model(model, vocab, test_ds, test_dl)
output_size, config.GRU_NUM_LAYERS, config.DROPOUT,
config.LEARNING_RATE, args.loss_function_type, dev)
print("Training the model...")
gru_model, loss_values = train_model(training_generator, gru_model,
criterion, optimizer, args.num_epochs,
config.BATCH_SIZE, dev,
args.loss_function_type)
# padded_sequences_train.size(1), dev, args.loss_function_type)
print("Plotting loss values...")
plot_loss(loss_values, args.loss_function_type)
print("Evaluating on validation data...")
lang_int_to_label_mapping = {
y: x
for x, y in lang_label_to_int_mapping.items()
}
test_model(gru_model, vocab_mapping, lang_int_to_label_mapping, X_test,
Y_test, dev)
print("Saving model to disk...")
joblib.dump(
gru_model, "{}_{}.pkl".format(config.GRU_MODEL_PATH,
args.loss_function_type))
joblib.dump(vocab_mapping, config.VOCAB_MAPPING)
joblib.dump(lang_label_to_int_mapping, config.LANG_LABEL_MAPPING)
# print("Testing the model...")
# test_model(gru_model, vocab_mapping, X_test, Y_test)
# This is the file you run to have the test results # In console, run: python run_test.py from test_model import test_model from Model import Model from Model import GenderModel # def model_1(a, b): # return a + b # # result = test_model(model_1) # # print "Result of model_1 is %d." % result model1 = Model() result = test_model(model1) model2 = GenderModel() result = test_model(model2)
treebanks.append((name, treebank))
# compute mspe and binary accuracy for models
results_all = dict()
for pref in prefs:
print "computing accuracies for model", pref
model_architecture = pref + '.json'
model_weights = pref + '_weights.h5'
model_dmap = 'models/dmap'
results = test_model(architecture=architecture,
model_architecture=model_architecture,
model_weights=model_weights,
dmap=model_dmap,
optimizer='adam',
metrics=metrics,
loss='mse',
digits=digits,
test_sets=treebanks,
classifiers=classifiers,
print_results=False)
results_all[pref] = results
# plot symbolic approaches
x = np.arange(1, 10)
metrics = results_all[prefs[0]].keys()
for metric in metrics:
if metric[-4:] == 'loss':
from load_data_train import train_images, train_labels
from load_data_test import test_images, test_labels
from load_parameters import update_parameters
from train_model import train_model
from test_model import test_model
train_model(train_images, train_labels)
weights = update_parameters()
print('Test model with data train.')
test_model(train_images, train_labels, weights)
print('Test model with data test.')
test_model(test_images, test_labels, weights)
BernNB = BernoulliNB()
BernNB.fit(X_train, y_train)#model eğitiliyor.
y_predict = BernNB.predict(X_test)
cm = confusion_matrix(y_test, y_predict) #Doğruluk matrisi. true negative false pozitive vs.
print("BERNOULLI")
print(cm)
print("BernoulliNB accuracy : " + str(BernNB.score(X_test,y_test)))#train kümeleri ile model eğitilmüşti. Şimdi eğitim ile gelen yaklaşımalr test setine uygulanıp accuracy hesaplanıyor.
MultiNB = MultinomialNB()
MultiNB.fit(X_train, y_train)
y_predict = MultiNB.predict(X_test)
cm = confusion_matrix(y_test, y_predict)
print("MultinomialNB")
print(cm)
print("MultinomialNB accuracy : " + str(MultiNB.score(X_test,y_test)))
GausNB = GaussianNB()
GausNB.fit(X_train, y_train)
y_predict = GausNB.predict(X_test)
cm = confusion_matrix(y_test, y_predict)
print("GaussianNB")
print(cm)
print("GaussianNB accuracy : " + str(GausNB.score(X_test,y_test)))
import test_model
print(test_model.test_model(GausNB))
def _test_domain_model(self, loaders, test_history):
model_f = self.models.model_f.eval()
model_c = self.models.model_c.eval()
model_d = self.models.model_d.eval()
source_test_loader = loaders.source_test_loader
target_test_loader = loaders.target_test_loader
merged_test_loader = loaders.merged_test_loader
domain_test_loss = 0
domain_correct = 0
with torch.no_grad():
class_model = nn.Sequential(model_f, model_c)
domain_model = nn.Sequential(model_f, model_d)
source_test_loss, source_correct = test_model.test_model(
class_model,
self.device,
self.criterions,
source_test_loader,
no_print=True)
target_test_loss, target_correct = test_model.test_model(
class_model,
self.device,
self.criterions,
target_test_loader,
no_print=True)
for data, target in merged_test_loader:
data = data.to(self.device)
if merged_test_loader.dataset.get_labels:
_, domains = target
else:
domains = target
domains = domains.to(self.device)
domain_out = domain_model(data)
domain_pred = domain_out.max(1, keepdim=True)[1]
domain_correct += domain_pred.eq(
domains.view_as(domain_pred)).sum().item()
domain_test_loss /= len(merged_test_loader.dataset)
test_history['target_loss'].append(target_test_loss)
test_history['source_loss'].append(source_test_loss)
test_history['target_acc'].append(100. * target_correct /
len(target_test_loader.dataset))
test_history['source_acc'].append(100. * source_correct /
len(source_test_loader.dataset))
test_history['domain_acc'].append(100. * domain_correct /
len(merged_test_loader.dataset))
if self.print_logs:
print(
'\nTarget Domain Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'
.format(
target_test_loss, target_correct,
len(target_test_loader.dataset),
100. * target_correct / len(target_test_loader.dataset)))
print(
'Source Domain Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'
.format(
source_test_loss, source_correct,
len(source_test_loader.dataset),
100. * source_correct / len(source_test_loader.dataset)))
print('Domains predictor: Accuracy: {}/{} ({:.0f}%)\n'.format(
domain_correct, len(merged_test_loader.dataset),
100. * domain_correct / len(merged_test_loader.dataset)))
return 100. * target_correct / len(target_test_loader.dataset)
# eğitilen modele uygun ağacı çizen fonksiyon.
def show_tree(tree, features, path):
f = io.StringIO()
sklearn.tree.export_graphviz(tree, out_file = f, feature_names=features)
pydotplus.graph_from_dot_data(f.getvalue()).write_png(path)
img=plt.imread(path)
plt.rcParams["figure.figsize"] = (20,20)
plt.imshow(img)
#jupyter de çalıştırınca yukardaki grrafik ile çakışmıyor ama spyder ide ile çakısıyor üst üste binen 2 foto çıkıyor.
#show_tree(dt, features, 'dec_tree_01.png')#yukardaki sebep nedeniyle ayrı ayrı parça parça yürütcem sunumda bu kısmı .
y_predict = dt.predict(x_test)
cm = confusion_matrix(y_test, y_predict)
print("SVM")
print(cm)
score = dt.score(x_test, y_test)
print("Score : " ,score)
import test_model
print(test_model.test_model(dt))
# This is the file you run to have the test results # In console, run: python run_test.py from test_model import test_model from Model import Model from Model import GenderModel # def model_1(a, b): # return a + b # # result = test_model(model_1) # # print "Result of model_1 is %d." % result model1 = Model() result = test_model(model1, 50) model2 = GenderModel() result = test_model(model2, 50)
model_params.model_path),
monitor="val_loss",
save_best_only=True)
model.fit_generator(
model_params.dataset.train_frame_generator(
model_params.frame_size, model_params.batch_size,
model_params.get_classifying()),
steps_per_epoch=model_params.nr_train_steps,
epochs=model_params.n_epochs,
validation_data=model_params.dataset.validation_frame_generator(
model_params.frame_size, model_params.batch_size,
model_params.get_classifying()),
validation_steps=model_params.nr_val_steps,
verbose=2,
callbacks=[
tensorboard_callback, plot_figure_callback, log_callback,
save_model_callback
])
print('Saving model and results...')
model.save(model_params.model_path + "/" + "final_model.h5")
print('\nDone!')
if __name__ == '__main__':
model_parameters = ModelTrainingParameters()
configure_gpu(model_parameters.gpu)
train_model(model_parameters)
test_model(model_parameters)
import numpy as np
import load_dataset as ld
import transformation as tfm
import test_model as tsm
import train_model as trm
import segmentation_models_pytorch as smp
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# Based on https://github.com/qubvel/segmentation_models.pytorch/blob/master/examples/cars%20segmentation%20(camvid).ipynb tutorial
# Load params from auxiliar class
import model_parameters as mp
from torch.utils.tensorboard import SummaryWriter
model_class = mp.model_params()
model_class.test_dir = 'Data/Test_images/'
# Run parameters
train_model = False
test_sample = True
print('semantic segmentation V1.0')
if train_model:
trm.train_new_model(model_class, 3, 0.3, True, 35)
if test_sample:
tsm.test_model(model_class)
print(args)
logging.info(args)
dataloaders, dataset_sizes, class_names = get_data_loaders(
args.train_data, args.batch_size)
logging.info("Train size {}, Val size {}, Test size {}".format(
dataset_sizes['train'], dataset_sizes['val'], dataset_sizes['test']))
logging.info('Class names:{}'.format(class_names))
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
total_gpus = torch.cuda.device_count()
logging.info('Total number of GPUs:{}'.format(total_gpus))
if total_gpus == 1:
multi_gpu = False
elif total_gpus > 1:
multi_gpu = True
else:
print("No GPUs, Cannot proceed. This training regime needs GPUs.")
exit(1)
nb_classes = len(class_names)
# Get a batch of training data and show it
inputs, classes = next(iter(dataloaders['train']))
out = torchvision.utils.make_grid(inputs)
show_batch(out, title=[class_names[x] for x in classes])
model, train_losses, val_losses = configure_run_model()
display_losses(train_losses, val_losses, 'Train-Val Loss')
test_model(model, dataloaders, device)
feed_dict={
birth_date:
birth_dates,
history:
np.array(list(np.array(df.history))),
current_products:
np.array(list(np.array(df.currentProducts))),
new_product:
np.array(list(np.array(df.newProducts))),
consumer_product:
np.array(list(np.array(df.ConsumerType)), dtype=float),
business_product:
np.array(list(np.array(df.BusinessType)), dtype=float),
training:
True
})
if i % 100 == 0:
print("train_loss: ", train_loss)
print(
"train accuracy: ",
test_model(sess, df, birth_date, history, current_products,
consumer_product, business_product, new_product,
loss, optimizer, out, training))
print(
"test accuracy: ",
test_model(sess, df_test, birth_date, history,
current_products, consumer_product,
business_product, new_product, loss, optimizer, out,
training))
saver.save(sess, "./checkpoint.ckpt")
model.compile(optimizer="adam",loss="binary_crossentropy", metrics=["accuracy"])
tmp = [i for i in range(0,len(train_data_output))]
random.shuffle(tmp)
train_data_input=[train_data_input[i] for i in tmp]
train_data_output=[train_data_output[i] for i in tmp]
print(np.asarray(train_data_input).shape)
print(np.asarray(train_data_output).shape)
for i in range(0,len(train_data_output)):
if train_data_output[i]==0:
train_data_output[i]=[1,0,0]
elif train_data_output[i]==1:
train_data_output[i]=[0,1,0]
elif train_data_output[i]==2:
train_data_output[i]=[0,0,1]
model.fit(np.asarray(train_data_input),np.asarray(train_data_output),epochs=100, verbose=2, batch_size=256)
# serialize model to JSON
model_json = model.to_json()
with open("./models/model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("./models/model.h5")
print("Saved model to disk")
test_model()
def index():
global file_choice
global init_file_choice
global cloud_1
global cloud_2
message = "Detect: " + file_choice
form = userInputForm()
l = ""
top_k_sent_words = []
bottom_k_sent_words = []
c = 0
if init_file_choice:
file_choice = "sentiment"
init_file_choice = False
if form.sent_analysis.data:
file_choice = "sentiment"
message = "Detect: Sentiment"
cloud_2 = "Top Negative Words"
cloud_1 = "Top Positive Words"
elif form.toxic_analysis.data:
file_choice = "toxic"
message = "Detect: Toxicity"
cloud_1 = "Top Toxic Words"
cloud_2 = "Top Non-Toxic Words"
elif form.validate_on_submit() and form.input_str.data:
input_text = str(form.input_str.data)
# print("BABYCAKES ROUND 2 \n\n\n")
# print(input_text)
input_text = input_text.lower().replace('out', ' ')
# print("BABYCAKES WAS HERE \n\n\n\n")
# print(input_text)
(l, c, top_k_words,
bottom_k_words) = test_model(input_text, file_choice)
# print("TOP K WORDS\n\n\n")
# print(top_k_words)
#
# print("BOTTOM K WORDS\n\n\n")
# print(bottom_k_words)
# TOP K top_k_words
toxic_words = ''
for tw in bottom_k_words:
# print(tw)
toxic_words = toxic_words + tw + ' '
non_toxic_words = ''
for tw in top_k_words:
# print(tw)
non_toxic_words = non_toxic_words + tw + ' '
stopwords = set(STOPWORDS)
non_toxic_cloud_words = ''
words_in_good_cloud = False
for w in input_text.split(' '):
if (w in non_toxic_words) and (w not in stopwords):
if w.strip():
desired_word = [d for d in top_k_words if w in d]
same_word = [w1 for w1 in desired_word if w1 == w]
if (len(same_word) > 0):
if (file_choice == "sentiment"):
top_k_words.reverse()
top_k_sent_words.append(
(w,
str(top_k_words.index(same_word[0])) + "th word"))
else:
top_k_sent_words.append(
(w, str(top_k_words.index(desired_word[0])) +
"th word"))
non_toxic_cloud_words = non_toxic_cloud_words + w + ' '
words_in_good_cloud = True
toxic_cloud_words = ''
words_in_bad_cloud = False
for w in input_text.split(' '):
print("this is w: " + w)
if (w in toxic_words) and (w not in stopwords) and (
w not in non_toxic_words):
if w.strip():
desired_word = [d for d in bottom_k_words if w in d]
same_word = [w1 for w1 in desired_word if w1 == w]
if (len(same_word) > 0):
if (file_choice == "toxic"):
bottom_k_words.reverse()
bottom_k_sent_words.append(
(w, str(bottom_k_words.index(same_word[0])) +
"th word"))
else:
bottom_k_sent_words.append(
(w, str(bottom_k_words.index(desired_word[0])) +
"th word"))
# print("BABYCAKES HIDIN HERE\n\n")
# print(w)
# print(len(w))
toxic_cloud_words = toxic_cloud_words + w + ' '
words_in_bad_cloud = True
# list index of sentence words in top list
#
# for w in input_text.strip().split(' '):
# desired_word = [d for d in top_k_words if w in d]
# if desired_word:
# top_k_sent_words.append((desired_word[0],top_k_words.index(desired_word[0])))
# else:
# bottom_k_sent_words.append((w,'Not in Top K words'))
if words_in_good_cloud:
# print("LETS SEE WHATS IN TOXI WORDS\n\n\n\n")
# print(toxic_cloud_words)
wordcloud = WordCloud(
width=512,
height=512,
background_color='black',
stopwords=stopwords,
min_font_size=10).generate(non_toxic_cloud_words)
wordCloud_good = 'static/images/wordCloud_toxic.png'
wordcloud.to_file(wordCloud_good)
else:
wordcloud = WordCloud(width=512,
height=512,
background_color='black',
stopwords=stopwords,
min_font_size=10).generate('None-found')
wordCloud_good = 'static/images/wordCloud_toxic.png'
wordcloud.to_file(wordCloud_good)
if words_in_bad_cloud:
# print("LETS SEE WHATS IN NON-TOXI WORDS\n\n\n\n")
# print(non_toxic_cloud_words)
wordcloud = WordCloud(width=512,
height=512,
background_color='black',
stopwords=stopwords,
min_font_size=10).generate(
str(toxic_cloud_words))
wordCloud_bad = 'static/images/wordCloud_non_toxic.png'
wordcloud.to_file(wordCloud_bad)
else:
wordcloud = WordCloud(width=512,
height=512,
background_color='black',
stopwords=stopwords,
min_font_size=10).generate('None-found')
wordCloud_bad = 'static/images/wordCloud_non_toxic.png'
wordcloud.to_file(wordCloud_bad)
wordCloud_bad_file = "images/wordCloud_toxic.png"
wordCloud_good_file = "images/wordCloud_non_toxic.png"
# if(file_choice == 'toxic'):
# temp = wordCloud_bad_file
# wordCloud_bad_file = wordCloud_good_file
# wordCloud_good_file = temp
# message = "Label: "+ str(l) +"-------- Confidence: "+ str(c)
return render_template('index.html',
topSentWords=top_k_sent_words,
botSentWords=bottom_k_sent_words,
detect=file_choice,
cloud_1=cloud_1,
cloud_2=cloud_2,
word_cloud_good=wordCloud_good_file,
wordCloud_bad=wordCloud_bad_file,
message=message,
form=form,
label=str(l),
confidence=int(c * 100),
str_conf=str(c))
