def init_vn(mark, homo_str):
D = NN_MEM_DEPTH
hidden_dims = HIDDEN_DIMS
degree = NN_DEGREE
if degree is None: degree = len(hidden_dims) + 1
elif degree < 1: raise ValueError('!! Degree must be greater than 1')
activation = lambda: Activation('relu')
learning_rate = 0.001
reg = None
# Initiate model
model = NeuralNet(D, mark, degree=degree, orders=NN_ORDERS)
for order in range(NN_MAX_VOL_ORD + 1, degree + 1):
if order not in NN_ORDERS: continue
dims = hidden_dims[order - NN_MAX_VOL_ORD - 1]
for dim in dims:
model.nn.add(order, Linear(dim, weight_regularizer='l2', strength=reg))
model.nn.add(order, activation())
model.nn.add(order, Linear(1, weight_regularizer='l2', strength=reg))
# Build model
model.nn.build(loss='euclid', metric='ratio', metric_name='Err %',
homo_strength=homo_str,
optimizer=tf.train.AdamOptimizer(learning_rate))
return model
def mlp_00(memory_depth, mark):
D = memory_depth
hidden_dims = [10, 10, 10]
activation = lambda: Activation('relu')
learning_rate = 0.001
reg = 0.00
# Initiate model
model = NeuralNet(memory_depth, mark)
model.nn.add(Input([D]))
for dim in hidden_dims:
model.nn.add(
Linear(output_dim=dim, weight_regularizer='l2', strength=reg))
model.nn.add(activation())
model.nn.add(Linear(output_dim=1, weight_regularizer='l2', strength=reg))
# Build model
model.nn.build(loss='euclid',
metric='ratio',
metric_name='Err%',
optimizer=tf.train.AdamOptimizer(learning_rate))
return model
def vn_00(memory_depth, mark, degree=None, homo_str=0.0):
D = memory_depth
hidden_dims = [[40] * 4, [40] * 5]
if degree is None: degree = len(hidden_dims) + 1
elif degree < 1: raise ValueError('!! Degree must be greater than 1')
activation = lambda: Activation('relu')
learning_rate = 0.001
reg = 0.00
reg = None
# Initiate model
model = NeuralNet(D, mark, degree=degree)
for order in range(2, degree + 1):
dims = hidden_dims[order - 2]
for dim in dims:
model.nn.T[order].add(
Linear(dim, weight_regularizer='l2', strength=reg))
model.nn.T[order].add(activation())
model.nn.T[order].add(Linear(1, weight_regularizer='l2', strength=reg))
# Build model
model.nn.build(loss='euclid',
metric='ratio',
metric_name='Err%',
homo_strength=homo_str,
optimizer=tf.train.AdamOptimizer(learning_rate))
return model
# endregion : Volterra Networks
def mlp00(mark, memory_depth, hidden_dim, learning_rate, activation):
# Initiate a neural net
model = NeuralNet(memory_depth, mark=mark)
nn = model.nn
# Add layers
nn.add(Input([memory_depth]))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate=learning_rate)
# Return model
return model
def main(grid): # Get Clean Data X, Y = read_clean_data() # Linear Regression try: LinearRegression(X, Y, grid) except Exception as e: print(e) # Binarize Y Y_binary = BinaryY(Y) # Logistic Regression try: LogisticRegression(X, Y_binary, grid) except Exception as e: print(e) # Decision Tree try: DecisionTree(X, Y_binary, grid) except Exception as e: print(e) # Support Vector Machine try: SVM(X, Y_binary, grid) except Exception as e: print(e) # Random Forest try: RandomForest(X, Y_binary, grid) except Exception as e: print(e) # Bagging Classifier try: Bagging(X, Y_binary, grid) except Exception as e: print(e) # Neural Network try: NeuralNet(X, Y_binary, grid) except Exception as e: print(e)
def mlp02(mark,
memory_depth,
branch_num,
hidden_dim,
learning_rate,
activation,
identity_init=False):
# Initiate a neural net
if identity_init:
model = NeuralNet(memory_depth,
mark=mark,
bamboo_braod=True,
identity_initial=True)
else:
model = NeuralNet(memory_depth,
mark=mark,
bamboo_broad=True,
identity_initial=False)
nn = model.nn
assert isinstance(nn, Bamboo_Broad)
# Add layers
nn.add(Input([memory_depth]))
branch = nn.add_branch()
branch.add(Linear(output_dim=hidden_dim))
branch.add(Activation(activation))
branch.add(Linear(output_dim=1))
for _ in range(branch_num - 1):
branch = nn.add_branch()
branch.add(
Linear(output_dim=hidden_dim,
weight_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer()))
branch.add(Activation(activation))
branch.add(
Linear(output_dim=1,
weight_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer()))
# Build model
model.default_build(learning_rate)
# Return model
return model
def mlp_res00(mark,
memory_depth,
branch_num,
hidden_dim,
learning_rate,
activation,
identity_init=True):
# Initiate a neural net
if identity_init:
model = NeuralNet(memory_depth,
mark=mark,
bamboo=True,
identity_initial=True)
else:
model = NeuralNet(memory_depth,
mark=mark,
bamboo=True,
identity_initial=False)
nn = model.nn
assert isinstance(nn, Bamboo)
# Add layers
nn.add(Input([memory_depth]))
for _ in range(branch_num):
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
branch = nn.add_branch()
branch.add(Linear(output_dim=1))
resnet = nn.add(ResidualNet())
resnet.add(Linear(output_dim=hidden_dim))
resnet.add(Activation(activation))
resnet.add(Linear(output_dim=hidden_dim))
resnet.add_shortcut()
resnet.add(Activation(activation))
nn.add(Linear(output_dim=hidden_dim))
nn.add(Activation(activation))
nn.add(Linear(output_dim=1))
# Build model
model.default_build(learning_rate)
# Return model
return model
class Solver(object):
def __init__(self, config):
# Configurations
self.config = config
# Build the models
self.build_models()
def build_models(self):
self.net = NeuralNet().to(self.config['device'])
self.optimizer = getattr(torch.optim, self.config['optimizer'])(
self.net.parameters(),
lr=self.config['lr'],
momentum=self.config['momentum'])
def save_model(self, filename):
save_path = os.path.join(self.save_path, f'{filename}')
try:
logging.info(
f'Saved best Neural network ckeckpoints into {save_path}')
torch.save(self.net.state_dict(),
save_path,
_use_new_zipfile_serialization=False)
except:
logging.error(f'Error saving weights to {save_path}')
def restore_models(self, filename):
weight_path = os.path.join(self.save_path, f'{filename}')
try:
logging.info(f'Loading the trained Extractor from {weight_path}')
self.extractor.load_state_dict(
torch.load(weight_path,
map_location=lambda storage, loc: storage))
except:
logging.error(f'Error loading weights from {weight_path}')
def train(self, tr_set, dv_set):
min_mse = 1000.
loss_record = {'train': [], 'dev': []}
early_stop_cnt = 0
epoch = 1
while epoch <= self.config['n_epochs']:
# Start training
self.net.train()
for i_batch, batch in enumerate(tr_set, start=1):
if i_batch != len(tr_set):
print(f" {i_batch+1}/{len(tr_set)} iters", end='\r')
else:
print(f" {i_batch+1}/{len(tr_set)} iters")
x, y = batch[0].to(self.config['device']), batch[1].to(
self.config['device'])
pred = self.net(x)
mse_loss = self.net(pred, y)
# Update model
self.optimzier.zero_grad()
mse_loss.backward()
self.optimzier.step()
# Record
loss_record['train'] += [mse_loss.detach().cpu().item()]
# After each epoch, test your model on the validation set
self.net.eval()
dev_loss = 0.0
for x, y in dv_set:
x, y = x.to(self.config['device']), y.to(self.config['device'])
with torch.no_grad():
pred = self.net(x)
mse_loss = self.net.cal_loss(pred, y)
dev_loss = mse_loss.detach().cpu().item() * len(x)
dev_loss = dev_loss / len(dv_set.dataset)
if dev_loss < min_mse:
# Save model if model improved
min_mse = dev_loss
print(
f'Saving model (epoch = {epoch:4d}, loss = {min_mse:.4f})')
self.save_model(f'min_mse.pt')
early_stop_cnt = 0
else:
early_stop_cnt += 1
epoch += 1
loss_record['dev'] += [dev_loss]
if early_stop_cnt > self.config['early_stop']:
# Stop training if your model stops imporving for "config['early_stop']" epochs.
break
print(f'Finished training after {epoch} epochs.')
return min_mse, loss_record
def test(self, te_set):
self.net.eval()
preds = []
for data in te_set:
x = data.to(self.config['device'])
with torch.no_grad():
pred = self.net(x)
preds += [pred.detach().cpu()]
preds = torch.cat(preds, dim=0).numpy()
return preds
def build_models(self):
self.net = NeuralNet().to(self.config['device'])
self.optimizer = getattr(torch.optim, self.config['optimizer'])(
self.net.parameters(),
lr=self.config['lr'],
momentum=self.config['momentum'])
device = torch.device("cuda") if torch.cuda.is_available() else torch.device(
"cpu")
args.span_range_height = args.span_range_width = args.span_range
args.grid_height = args.grid_width = args.grid_size # 4
args.image_height = args.image_width = 28
n_iter = 0
n_iter_val = 0
# create model
if args.model == 'no_stn':
print('create model without ClsNet')
model = ClsNet().to(device)
elif args.model == 'bp':
print('create model bp')
model = NeuralNet().to(device)
else:
print('create model with STN')
model = STNClsNet(args).to(device)
#model = torch.nn.DataParallel(model)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
train_set = datasets.MNIST(
'/home/roit/datasets/mnist/',
train=True,
download=True,
transform=transforms.Compose([
transforms.Lambda(lambda image: image.rotate(random.random() * args.
angle * 2 - args.angle)),
def __getitem__(self, index):
return self.x_data[index], self.y_data[index]
# we can call len(dataset) to return the size
def __len__(self):
return self.n_samples
dataset = ChatDataset()
train_loader = DataLoader(dataset=dataset,
batch_size=batch_size,
shuffle=True,
num_workers=0)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = NeuralNet(input_size, hidden_size, output_size).to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
# Train the model
for epoch in range(num_epochs):
for (words, labels) in train_loader:
words = words.to(device)
labels = labels.to(dtype=torch.long).to(device)
# Forward pass
outputs = model(words)
# if y would be one-hot, we must apply
# labels = torch.max(labels, 1)[1]
def train_and_predict(X_train_all, y_train_all, X_test, seed_num):
model_params["seed"] = seed + seed_num
oof_df = pd.DataFrame(
index=[i for i in range(X_train_all.shape[0])],
columns=[i for i in range(model_params["num_class"])])
y_preds = []
models = []
auc_scores = []
acc_scores = []
logloss_scores = []
kf = StratifiedKFold(n_splits=config["fold"],
shuffle=True,
random_state=model_params["seed"])
for fold_num, (train_index,
valid_index) in enumerate(kf.split(X_train_all,
y_train_all)):
logger.debug(f"FOLD: {fold_num}")
X_train, X_valid = (X_train_all.iloc[train_index, :],
X_train_all.iloc[valid_index, :])
y_train, y_valid = (y_train_all.iloc[train_index],
y_train_all.iloc[valid_index])
# train & inference
if model_name == "lightgbm":
classifier = LightGBM()
elif model_name == "nn":
classifier = NeuralNet(seed_num, fold_num)
elif model_name == "cnn1d":
classifier = CNN1d(seed_num, fold_num)
elif model_name == "logistic_regression":
classifier = LogisticRegressionClassifier()
else:
logger.debug("No such model name")
raise Exception
if "sampling" in config:
if config["sampling"] == "SMOTE":
X_train, y_train = SMOTE().fit_resample(X_train, y_train)
elif config["sampling"] == "ADASYN":
X_train, y_train = ADASYN().fit_resample(X_train, y_train)
elif config["sampling"] == "RandomOverSampler":
X_train, y_train = RandomOverSampler().fit_resample(
X_train, y_train)
else:
raise
y_pred, y_valid_pred, model = classifier.train_and_predict(
X_train, X_valid, y_train, y_valid, X_test, model_params)
# 結果の保存
y_preds.append(y_pred)
oof_df.iloc[valid_index, :] = y_valid_pred
models.append(model)
# スコア
auc_valid = evaluate_score(y_valid, y_valid_pred[:, 1], "auc")
acc_valid = evaluate_score(y_valid, y_valid_pred.argmax(axis=1), "acc")
logloss_valid = evaluate_score(y_valid, y_valid_pred[:, 1], "logloss")
logger.debug(
f"\t auc:{auc_valid}, acc: {acc_valid}, logloss: {logloss_valid}")
auc_scores.append(auc_valid)
acc_scores.append(acc_valid)
logloss_scores.append(logloss_valid)
# lightgbmなら重要度の出力
if model_name == "lightgbm":
feature_imp_np = np.zeros(X_train_all.shape[1])
for model in models:
feature_imp_np += model.feature_importance() / len(models)
feature_imp = pd.DataFrame(sorted(
zip(feature_imp_np, X_train_all.columns)),
columns=['Value', 'Feature'])
#print(feature_imp)
logger.debug(feature_imp)
plt.figure(figsize=(20, 10))
sns.barplot(x="Value",
y="Feature",
data=feature_imp.sort_values(by="Value", ascending=False))
plt.title('LightGBM Features (avg over folds)')
plt.tight_layout()
plt.savefig(f'./logs/plots/features_{config_filename}.png')
# CVスコア
auc_score = sum(auc_scores) / len(auc_scores)
acc_score = sum(acc_scores) / len(acc_scores)
logloss_score = sum(logloss_scores) / len(logloss_scores)
logger.debug('=== CV scores ===')
logger.debug(
f"\t auc:{auc_score}, acc: {acc_score}, logloss: {logloss_score}")
# submitファイルの作成
sub = pd.DataFrame(pd.read_feather(f'data/interim/test.feather')[ID_name])
y_sub = sum(y_preds) / len(y_preds)
sub[target_name] = y_sub[:, 1]
''' 確率ではなく番号を出力
if y_sub.shape[1] > 1:
y_sub = np.argmax(y_sub, axis=1)
'''
return oof_df, sub
batch_size=args.valid_batch_size,
shuffle=False,
drop_last=False)
if args.model_type == "LSTM":
max_features = None
max_features = max_features or len(tokenizer.word_index) + 1
if args.embedding_type == "baseline":
# Loading weight embedding Glove
glove_matrix, unknown_words_glove = build_matrix(
tokenizer.word_index, args.glove_embedding_path,
args.embedding_dim, max_features)
print('n unknown words (glove): ', len(unknown_words_glove))
model = NeuralNet(
torch.FloatTensor(glove_matrix).cpu(), args.lstm_units)
model.zero_grad()
model.to(device)
elif args.embedding_type == "coded":
# Load GloVE embeddings
orig_embeddings = torch.load(args.data_folder + 'all_orig_emb.pt')
# Load shared words and all GloVE words
with open(args.data_folder + "shared_words.txt", "r") as file:
shared_words = file.read().split('\n')
with open(args.data_folder + "glove_words.txt", "r") as file:
glove_words = file.read().split('\n')
# Recreate GloVE_dict
glove_dict = {}
for i, word in enumerate(glove_words):
glove_dict[word] = orig_embeddings[i]
def main(grid):
# Get Clean Data
X, Y = read_clean_data()
Y_binary = BinaryY(Y)
NeuralNet(X, Y_binary, grid)
def main():
args = parser.parse_args()
# seed everything to ensure reproducible results from different runs
if args.seed is not None:
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
###########################################################################
# Model
###########################################################################
global best_acc1
# create model
if args.arch == 'LogisticRegression':
model = LogisticRegression(input_size=13, n_classes=args.classes)
elif args.arch == 'NeuralNet':
model = NeuralNet(input_size=13, hidden_size=[32, 16], n_classes=args.classes) #hidden_size=[64, 32]
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
torch.cuda.set_device(args.gpu)
torch.backends.cudnn.benchmark = True
model = model.cuda(args.gpu)
# print(model)
if args.train_file:
print(30 * '=')
print(summary(model, input_size=(1, 13),
batch_size=args.batch_size, device='cpu'))
print(30 * '=')
###########################################################################
# save directory
###########################################################################
save_dir = os.path.join(os.getcwd(), args.save_dir)
save_dir += ('/arch[{}]_optim[{}]_lr[{}]_lrsch[{}]_batch[{}]_'
'WeightedSampling[{}]').format(args.arch,
args.optim,
args.lr,
args.lr_scheduler,
args.batch_size,
args.weighted_sampling)
if args.suffix:
save_dir += '_{}'.format(args.suffix)
save_dir = save_dir[:]
if not os.path.exists(save_dir):
os.makedirs(save_dir)
###########################################################################
# Criterion and optimizer
###########################################################################
# Initialise criterion and optimizer
if args.gpu is not None:
criterion = nn.CrossEntropyLoss().cuda(args.gpu)
else:
criterion = nn.CrossEntropyLoss()
# define optimizer
print("=> using '{}' optimizer".format(args.optim))
if args.optim == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
else: # default is adam
optimizer = torch.optim.Adam(model.parameters(), args.lr,
betas=(0.9, 0.999), eps=1e-08,
weight_decay=args.weight_decay,
amsgrad=False)
###########################################################################
# Resume training and load a checkpoint
###########################################################################
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
if args.gpu is None:
checkpoint = torch.load(args.resume)
else:
# Map model to be loaded to specified single gpu.
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.resume, map_location=loc)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_acc1']
if args.gpu is not None:
# best_acc1 may be from a checkpoint from a different GPU
best_acc1 = best_acc1.to(args.gpu)
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
###########################################################################
# Data Augmentation
###########################################################################
# TODO
###########################################################################
# Learning rate scheduler
###########################################################################
print("=> using '{}' initial learning rate (lr)".format(args.lr))
# define learning rate scheduler
scheduler = args.lr_scheduler
if args.lr_scheduler == 'reduce':
print("=> using '{}' lr_scheduler".format(args.lr_scheduler))
# Reduce learning rate when a metric has stopped improving.
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.5,
patience=10)
elif args.lr_scheduler == 'cyclic':
print("=> using '{}' lr_scheduler".format(args.lr_scheduler))
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer,
base_lr=0.00005,
max_lr=0.005)
elif args.lr_scheduler == 'cosine':
print("=> using '{}' lr_scheduler".format(args.lr_scheduler))
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=100,
eta_min=0,
last_epoch=-1)
###########################################################################
# load train data
###########################################################################
if args.train_file:
train_dataset = HeartDiseaseDataset(csv=args.train_file, label_names=LABELS)
if args.weighted_sampling:
train_sampler = torch.utils.data.WeightedRandomSampler(train_dataset.sample_weights,
len(train_dataset),
replacement=True)
else:
train_sampler = None
###########################################################################
# update criterion
print('class_sample_count ', train_dataset.class_sample_count)
print('class_probability ', train_dataset.class_probability)
print('class_weights ', train_dataset.class_weights)
print('sample_weights ', train_dataset.sample_weights)
if args.weighted_loss:
if args.gpu is not None:
criterion = nn.CrossEntropyLoss(weight=train_dataset.class_weights).cuda(args.gpu)
else:
criterion = nn.CrossEntropyLoss(weight=train_dataset.class_weights)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size, shuffle=(train_sampler is None),
num_workers=args.workers, pin_memory=True, sampler=train_sampler)
###########################################################################
# load validation data
###########################################################################
if args.valid_file:
valid_dataset = HeartDiseaseDataset(csv=args.valid_file, label_names=LABELS)
val_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
if args.evaluate:
# retrieve correct save path from saved model
save_dir = os.path.split(args.resume)[0]
validate(val_loader, model, criterion, save_dir, args)
return
###########################################################################
# Train the model
###########################################################################
for epoch in range(args.start_epoch, args.epochs):
# adjust_learning_rate(optimizer, epoch, args)
print_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer,
scheduler, epoch, args)
# evaluate on validation set
acc1 = validate(val_loader, model, criterion, save_dir, args)
# update learning rate based on lr_scheduler
if args.lr_scheduler == 'reduce':
scheduler.step(acc1)
elif args.lr_scheduler == 'cosine':
scheduler.step()
# remember best acc@1 and save checkpoint
is_best = acc1 >= best_acc1
best_acc1 = max(acc1, best_acc1)
print("Saving model [{}]...".format(save_dir))
save_checkpoint({'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optimizer.state_dict(),
'criterion': criterion, },
is_best,
save_dir=save_dir)
print(30 * '=')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
with open('intents.json', 'r') as json_data:
intents = json.load(json_data)
FILE = "data.pth"
data = torch.load(FILE)
input_size = data["input_size"]
hidden_size = data["hidden_size"]
output_size = data["output_size"]
all_words = data['all_words']
tags = data['tags']
model_state = data["model_state"]
model = NeuralNet(input_size, hidden_size, output_size).to(device)
model.load_state_dict(model_state)
model.eval()
bot_name = "D.Ace"
print("\t\t\tAI Open Ended Project")
print("\t\t\t By Sanskar Sharma")
print("\t\t\t PRN: 0120180381\n")
print("Let's chat! (type 'quit' to exit)")
name = input("D.Ace: what is your good name?\n\nYou: ")
val = random.choice([
". Such a nice name", ". What do you wanna talk about?",
". We're one family now."
])
print("\nD.Ace: Welcome, ", name, val, sep='')
while True: