def restore_model(file_path: str):
info = torch.load(file_path)
arch: str = info['hparams'].arch
batch_norm = info['hparams'].batch_norm
dataset: str = info['hparams'].dataset
hidden_layers: int = info['hparams'].hidden_layers
hidden_size: int = info['hparams'].hidden_size
if arch == 'mlp' and dataset == 'mnist':
model: nn.Module = MLP(input_size=784,
hidden_size=hidden_size,
num_hidden_layers=hidden_layers,
batch_norm=batch_norm)
elif arch == 'alexnet':
model: nn.Module = AlexNet()
else:
model: nn.Module = MLP(hidden_size=hidden_size,
num_hidden_layers=hidden_layers,
batch_norm=batch_norm)
model.load_state_dict(info['model_state_dict'])
lr = info['hparams'].lr
momentum = info['hparams'].momentum
weight_decay = info['hparams'].weight_decay
optimizer = optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=weight_decay)
optimizer.load_state_dict(info['optimizer_state_dict'])
# model.eval()
return model, optimizer, info['hparams']
def configure(**kwargs):
"""Configure critic."""
observation_space = kwargs['observation_space']
action_space = kwargs['action_space']
assert isinstance(observation_space, Box)
nb_state_feats = observation_space.shape[-1]
net_dict = dict(nb_layers=kwargs['nb_layers'],
hidden_size=kwargs['hidden_size'])
val_function = MLP(nb_inputs=nb_state_feats, nb_outputs=1, **net_dict)
if isinstance(action_space, Discrete):
nb_actions = action_space.n
adv_function = MLP(nb_inputs=nb_state_feats,
nb_outputs=nb_actions,
**net_dict)
elif isinstance(action_space, Box):
nb_actions = action_space.shape[-1]
adv_function = ContinuousAdvantageMLP(
nb_outputs=1,
nb_state_feats=nb_state_feats,
nb_actions=nb_actions,
**net_dict)
if kwargs['normalize']:
val_function = NormalizedMLP(val_function)
adv_function = NormalizedMLP(adv_function)
return AdvantageCritic(kwargs['dt'], kwargs['gamma'], kwargs['lr'],
kwargs['tau'], kwargs['optimizer'],
val_function, adv_function)
def configure_model(self):
"""
:return:
"""
arch: str = self.hparams.arch
batch_norm = self.hparams.batch_norm
dataset: str = self.hparams.dataset
hidden_layers: int = self.hparams.hidden_layers
hidden_size: int = self.hparams.hidden_size
if arch == 'mlp':
if dataset == 'mnist':
return MLP(input_size=784,
hidden_size=hidden_size,
num_hidden_layers=hidden_layers,
batch_norm=batch_norm)
elif dataset == 'cifar10':
return MLP(hidden_size=hidden_size,
num_hidden_layers=hidden_layers,
batch_norm=batch_norm)
else:
raise ValueError('invalid dataset specification!')
elif arch == 'alexnet':
return AlexNet()
elif arch == 'vgg11':
return VGG(vgg_name='VGG11')
elif arch == 'vgg13':
return VGG(vgg_name='VGG13')
elif arch == 'resnet18':
return ResNet18()
elif arch == 'resnet34':
return ResNet34()
else:
raise ValueError('Unsupported model!')
def estimate_divergences(model_hyperparams, lr, criterion, device, batch_size, n_epochs, is_wasserstein):
input_size, h1_size, h2_size, out_size, out_sigmoid = model_hyperparams
p_dist = iter(samplers.distribution1(x=0, batch_size=batch_size))
# train a model for each value of phi
phi_list = np.linspace(-1, 1, 21)
jsd_list = []
for phi in phi_list:
# create model and optimizer
model = MLP(input_size, h1_size, h2_size, out_size, out_sigmoid).to(device)
print(model)
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
q_dist = iter(samplers.distribution1(x=phi, batch_size=batch_size))
losses = train(model, p_dist, q_dist, optimizer, criterion, device, n_epochs, is_wasserstein)
# visualise loss.
# plt.figure()
# plt.plot(losses)
# plt.title('phi = {}'.format(phi))
# plt.show()
divergence_estimate = -1 * losses[-1]
print('At phi = {}, divergence estimate = {}'.format(phi, divergence_estimate))
jsd_list.append(divergence_estimate)
plt.figure()
plt.plot(phi_list, jsd_list, 'o')
plt.xlabel('$\phi$', fontsize=14)
plt.ylabel('Jensen-Shannon Divergence', fontsize=14)
plt.show()
def __init__(self, args, num_users, num_items):
BaseModel.__init__(self, args, num_users, num_items)
self.layers = eval(args.layers)
self.lambda_layers = eval(args.reg_layers)
self.num_factors = args.num_factors
self.model_GMF = GMF(args, num_users, num_items)
self.model_MLP = MLP(args, num_users, num_items)
def main():
save_dir = './imgs/'
TRAIN_RANGE[-1] += 1
TEST_RANGE[-1] += 1
# datasets
train_data = torch.arange(*TRAIN_RANGE).unsqueeze_(1).float()
test_data = torch.arange(*TEST_RANGE).unsqueeze_(1).float()
# train
all_mses =[]
for non_lin in NON_LINEARITIES:
print("Working with {}...".format(non_lin))
mses = []
for i in range(100):
net = MLP(4, 1, 8, 1, non_lin)
optim = torch.optim.RMSprop(net.parameters(), lr=LEARNING_RATE)
train(net, optim, train_data, NUM_ITERS)
mses.append(test(net, test_data))
all_mses.append(torch.cat(mses, dim=1).mean(dim=1))
all_mses = [x.numpy().flatten() for x in all_mses]
# plot
fig, ax = plt.subplots(figsize=(8, 7))
x_axis = np.arange(-20, 21)
for i, non_lin in enumerate(NON_LINEARITIES):
ax.plot(x_axis, all_mses[i], label=non_lin)
plt.grid()
plt.legend(loc='best')
plt.ylabel('Mean Absolute Error')
plt.savefig(save_dir + 'extrapolation.png', format='png', dpi=300)
plt.show()
def __init__(self,
dataset,
pretrained_net=None,
x_dim=1,
y_dim=1,
h_dim=32,
lr=0.1,
bs=8,
device='cpu',
acq_scheme='rand'):
super(PseudoBO, self).__init__()
self.R = dataset.inquiry
self.device = device
self.inn_lr = lr
self.bs = bs
self.acq_scheme = acq_scheme
self.net = MLP(x_dim, y_dim,
h_dim) if pretrained_net is None else pretrained_net
self.net.to(device)
# init w_list
self.inn_grad_step()
# init D_0
self.D = []
s = dataset.data[0].to(self.device)
self.D.append(s)
def train_model(train_tcrs, train_pathologies, test_tcrs, test_pathologies,
device, args, params):
"""
Train and evaluate the model
"""
losses = []
# We use Cross-Entropy loss
loss_function = nn.CrossEntropyLoss()
# Set model with relevant parameters
model = MLP(params['enc_dim'], params['out_dim'], device)
# Move to GPU
model.to(device)
# We use Adam optimizer
optimizer = optim.Adam(model.parameters(), lr=params['lr'], weight_decay=params['wd'])
# Train several epochs
for epoch in range(params['epochs']):
print('epoch:', epoch + 1)
# Train model and get loss
loss = train_epoch(train_tcrs, train_pathologies, model, loss_function, optimizer, device)
losses.append(loss)
# evaluate
acc = evaluate(model, train_tcrs, train_pathologies, device)
print('train accuracy:', acc)
acc = evaluate(model, test_tcrs, test_pathologies, device)
print('test accuracy:', acc)
return model
def main():
test_seen_loader = torch.utils.data.DataLoader(AttributeDataset(
args.data_dir,
args.dataset,
features_path=args.gan_path,
mode='test_seen',
generalized=True,
normalize=args.normalize,
sentences=args.sentences),
batch_size=args.batch_size,
shuffle=False)
test_unseen_loader = torch.utils.data.DataLoader(
AttributeDataset(args.data_dir,
args.dataset,
features_path=args.gan_path,
mode='test_unseen',
generalized=True,
normalize=args.normalize,
sentences=args.sentences),
batch_size=args.batch_size,
shuffle=False)
# instanciate the models
if args.mlp:
mlp = MLP(args.dim_input, [args.nhidden * 2], args.nhidden)
else:
mlp = LinearProjection(args.dim_input, args.nhidden)
embed = LinearProjection(args.nhidden, args.dim_embed)
if args.sentences:
cam_key = 'sentences'
else:
cam_key = 'emb'
if args.gan_path is not None:
cam_key = 'full_' + cam_key
cam = torch.from_numpy(test_seen_loader.dataset.data[cam_key].T)
proxies = ProxyNet(args.n_classes, args.dim_embed, proxies=cam)
model = Base(mlp, embed, proxies)
criterion = ProxyLoss(temperature=args.temp)
if args.cuda:
mlp.cuda()
embed.cuda()
model.cuda()
proxies.cuda()
# loading
checkpoint = torch.load(args.model_path)
model.load_state_dict(checkpoint['state_dict'])
txt = ("=> loaded checkpoint '{}' (epoch {})".format(
args.model_path, checkpoint['epoch']))
print(txt)
compute_scores(test_seen_loader, test_unseen_loader, model, criterion)
def main():
__model = MLP(in_dim=400, hidden_dim=HIDDEN_DIM, out_dim=800)
print("\tTraining {}...".format(__model.__str__().split("(")[0]))
optim = torch.optim.Adam(__model.parameters(), lr=LEARNING_RATE)
train(__model, optim, X_train, y_train, 128)
mse = test(__model, X_test, y_test).mean().item()
print(mse)
def check_model(self, X_train, X_val, y_train, y_val, X_test, y_test,
raw_seq):
""" Funtion used to navigate to the specific model. The is defined when initialising the class.
Reads the self.model_type
Each statement does the following:
- Calls function to format data for the model
- Calls funtion to train the model
- Calls funtion to plot the MSE graph
- Calls funtion to test the model
- Returns the accuarcy as R2 score"""
if self.model_type == 'CNN':
X_train, X_val, y_train, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4 = CNN.data_format(
X_train, X_val, y_train)
history = CNN.CNN_train_model(self, X_train, X_val, y_train, y_val,
self.verbose, n_input, n_output,
ytrain1, ytrain2, ytrain3, ytrain4)
Models.plotting(history)
yhat = CNN.CNN_test_model(self, X_test, self.verbose, y_test)
Models.accuracy(self, yhat, y_test, X_test, self.model_type)
if self.model_type == 'MLP':
X_train, X_val, y_train, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4 = MLP.data_format(
X_train, X_val, y_train)
history = MLP.MLP_train_model(self, X_train, X_val, y_train, y_val,
self.verbose, n_input, n_output,
ytrain1, ytrain2, ytrain3, ytrain4)
# Models.plotting(history)
yhat, final_cols = MLP.MLP_test_model(X_test, self.verbose, y_test)
Models.accuracy(self, yhat, y_test, final_cols, self.model_type)
if self.model_type == 'KNN':
X_train, X_val, y_train, X_test = KNN.data_format(
X_train, X_val, y_train, X_test)
yhat, final_cols = KNN.KNN_train_model(self, X_train, X_val,
y_train, y_val, X_test,
y_test, raw_seq)
Models.accuracy(self, yhat, y_test, final_cols, self.model_type)
if self.model_type == 'LSTM':
history, model = LSTMs.LSTM_train_model(self, X_train, X_val,
y_train, y_val,
self.verbose)
Models.plotting(history)
yhat = LSTMs.LSTM_test_model(X_test, model, self.verbose, y_test)
Models.accuracy(self, yhat, y_test, X_test, self.model_type)
if self.model_type == 'BASELINE':
n_input, X_train, n_output = BaseLine.data_format(X_train, y_train)
model = BaseLine.baseline_train(self, X_train, y_train, n_input,
n_output)
yhat, final_cols = BaseLine.baseline_test(X_test, n_input, model)
Models.accuracy(self, yhat, y_test, final_cols, self.model_type)
def main(args):
print('[*] Arguments: %s' % args)
print('[*] Read MNIST...')
num_test_images = args.num_test_images
images, labels = load_mnist('test', path='./data', max_ind=num_test_images)
images, labels = images[:num_test_images, :, :], labels[:num_test_images]
images = images.astype(np.float32)
images = images / 255.
print('[*] The shape of image: %s' % str(images.shape))
print('[*] Load the network...')
if args.network == 'mlp': # Lab 2
if args.quantized:
print('[!] MLP does not support quantization')
return
model_path = os.path.join('./pretrained_weights',
'mlp_iter_10000.caffemodel')
net = MLP(model_path, args)
elif args.network == 'cnn':
if args.quantized: # Lab 14
model_path = os.path.join('./pretrained_weights',
'quantized_cnn_weights.txt')
else: # Lab 11
model_path = os.path.join('./pretrained_weights',
'cnn_weights.txt')
net = CNN(model_path, args)
else:
raise
print('[*] Run tests...')
test_images = [images[i, :, :].copy() for i in xrange(num_test_images)]
n_correct = 0
start_time = time.time()
for i in xrange(num_test_images):
X = test_images[i]
X = X.reshape((28 * 28)) # 28x28->784
logit = net.inference(X)
prediction = np.argmax(logit)
label = labels[i, ]
n_correct += (label == prediction)
print('[*] Statistics...')
model_stats = {
'total_time': time.time() - start_time,
'total_image': num_test_images,
'accuracy': float(n_correct) / num_test_images,
'avg_num_call': net.total_num_call[0] / num_test_images,
'm_size': net.m_size,
'v_size': net.v_size,
}
pp.pprint(model_stats)
def main():
# MNIST dataset
train_dataset = torchvision.datasets.MNIST(
root='/Datadisk/jmlu/Pytorch_Data/',
train=True,
transform=transforms.ToTensor(),
download=False)
test_dataset = torchvision.datasets.MNIST(
root='/Datadisk/jmlu/Pytorch_Data/',
train=False,
transform=transforms.ToTensor())
# Data loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
model = MLP(input_size, hidden_size, num_classes).to(device)
print(model)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
if Baseline:
print("Baselie Training ----------------")
train(model, train_loader, num_epochs, device, optimizer, criterion)
test(model, test_loader, device)
# Save the model checkpoint
torch.save(model.state_dict(), 'model_base.ckpt')
exit()
if Prune:
print("Model Pruning -------------------")
model.load_state_dict(torch.load('model_base.ckpt'))
model_prune(model)
print('\nTesting After Prune: ')
test(model, test_loader, device)
torch.save(model.state_dict(), 'model_pruned.ckpt')
exit()
if Retrain:
print("Model Retrain -------------------")
model.load_state_dict(torch.load('model_pruned.ckpt'))
model_finetune(model, train_loader, num_epochs, device, optimizer,
criterion)
test(model, test_loader, device)
torch.save(model.state_dict(), 'model_finetuned.ckpt')
exit()
def __init__(self, args):
self.args = args
self.model = MLP(args.nin, args.nh, args.nout, args.do)
self.model.to(args.d)
print("model params {}".format(count_parameters(self.model)))
log_path = "logs/gmm"
if os.path.exists(log_path) and os.path.isdir(log_path):
shutil.rmtree(log_path)
self.writer = SummaryWriter(log_path)
nc = 2 if args.dataset == "toy" else 3
self.best_loss = np.inf
self.softmax = torch.nn.Softmax(dim=1)
self.ce = torch.nn.CrossEntropyLoss()
def train_minibatch():
generator = MiniBatchGenerator(861, x_mapper, y_mapper)
generator.split_train_test()
"""
print('load train mini-batch')
while True:
X, y = generator.load_next_train_batch(10)
if X is None:
break
print(y)
X_to_train, y_to_train = preprocess(X, y)
print(X_to_train.shape)
print(y_to_train.shape)
break
sess, y_predict, x_train, y_train = models.train_nn(NUM_CATEGORIES,
X_to_train, y_to_train,
layers=(256, 256),
iterations=1000)
print('load test mini-batch')
while True:
X, y = generator.load_next_test_batch(10)
if X is None:
break
X_to_test, y_to_test = preprocess(X, y)
correct_prediction = tf.equal(tf.argmax(y_predict, 1), tf.argmax(y_train, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print(sess.run(accuracy, feed_dict={x_train: X_to_test, y_train: y_to_test}))
correct_prediction = tf.equal(tf.argmax(y_predict, 1), tf.argmax(y_train, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#print(sess.run(accuracy, feed_dict={x_train: X_to_train, y_train: y_to_train}))
print(sess.run(accuracy, feed_dict={x_train: X_to_test, y_train: y_to_test}))
"""
model = MLP(NUM_CATEGORIES,
min_pc,
layers=(256, 256, 256),
learning_rate=0.0005)
model.start_session()
X_test, y_test = generator.load_next_test_batch(1000)
X_test, y_test = preprocess(X_test, y_test)
#sess = None
for iteration in range(1000):
generator.reset()
while True:
X_train, y_train = generator.load_next_train_batch(10)
if X_train is None:
break
X_train, y_train = preprocess(X_train, y_train)
model.train(X_train, y_train)
if iteration % 100 == 0:
print('loss = ', model.calculate_loss())
# print('train acc = ', model.calculate_accuracy(X_train, y_train))
print('test acc = ', model.calculate_accuracy(X_test, y_test))
print("\n---\n")
def mc_mlp_val(x_train,
y_train,
size_input,
hidden_layers,
nodes_hidden_layers,
size_output=2,
runs=1000,
n_samples=5):
max_acc = 0.0
max_conf = []
es = EarlyStopping(monitor='val_loss',
mode='min',
verbose=0,
patience=0,
restore_best_weights=True)
mlp_cache = {}
for _ in tqdm(range(runs)):
# generate random mlp
conf = gen_random_mlp(size_input, hidden_layers, nodes_hidden_layers,
size_output)
if conf not in mlp_cache:
acc = 0.0
for _ in range(n_samples):
model = MLP(conf, use_bias_input=False)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
epochs=50,
batch_size=300,
validation_split=0.2,
callbacks=[es],
verbose=0)
acc += history.history['val_accuracy'][-1]
acc /= n_samples
mlp_cache[conf] = acc
if acc > max_acc:
max_acc = acc
max_conf = conf
return max_conf, max_acc
def set_up_predictor(fp_out_dim,
net_hidden_dims,
class_num,
sim_method='mlp',
symmetric=None):
sim_method_dict = {
'mlp': 'multi-layered perceptron',
'ntn': 'bilinear transform',
'symmlp': 'symmetric perceptron',
'hole': 'holographic embedding',
'dist-mult': 'dist-mult',
}
logging.info('Link Prediction: {}'.format(
sim_method_dict.get(sim_method, None)))
lp = None
if sim_method == 'mlp':
lp = MLP(out_dim=class_num, hidden_dims=net_hidden_dims)
elif sim_method == 'ntn':
ntn_out_dim = 8
lp = NTN(left_dim=fp_out_dim,
right_dim=fp_out_dim,
out_dim=class_num,
ntn_out_dim=ntn_out_dim,
hidden_dims=net_hidden_dims)
elif sim_method == 'symmlp':
lp = MLP(out_dim=class_num, hidden_dims=net_hidden_dims)
elif sim_method == 'hole':
lp = HolE(out_dim=class_num, hidden_dims=net_hidden_dims)
elif sim_method == 'dist-mult':
dm_out_dim = 8
lp = DistMult(left_dim=fp_out_dim,
right_dim=fp_out_dim,
out_dim=class_num,
dm_out_dim=dm_out_dim,
hidden_dims=net_hidden_dims)
else:
raise ValueError(
'[ERROR] Invalid link prediction model: {}'.format(sim_method))
predictor = DDIPredictor(lp, symmetric=symmetric)
return predictor
def main(args):
exp_info = exp_config.Experiment(args.dataset)
paths = exp_info.paths
args.paths = paths
args.metadata = exp_info.metadata
np.random.seed(args.seed)
torch.manual_seed(args.seed)
args.batch_size = 1
feature_size, train_loader, val_loader, test_loader, all_loader = exp_info.get_dataset(
args, save=True)
label_num = exp_info.get_label_num(args)
hidden_size = 256
hidden_layers = 2
if args.model == 'lstm':
parsing_model = lstm_model.BiLSTM(feature_size, hidden_size,
hidden_layers, label_num)
else:
parsing_model = mlp_model.MLP(feature_size, hidden_size, label_num)
parsing_model = torch.nn.DataParallel(parsing_model)
prev = args.subsample
args.subsample = 1
args.save_path = os.path.join(paths.inter_root, 'likelihood', args.task,
args.model)
args.resume = os.path.join(
paths.checkpoint_root,
'detection_{}_{}_e{}_lr{}_b{}_lrd{}_s{}_do{}'.format(
args.task, args.model, args.epochs, args.lr, args.using_batch_size,
args.lr_decay, 1 if not args.subsample else args.subsample,
args.dropout_rate))
args.subsample = prev
logutils.load_checkpoint(args, parsing_model)
validate(test_loader, parsing_model, args=args)
def __init__(self, host, port):
if model == 'cnn':
if dataset == 'mnist' or dataset == 'fmnist':
self.global_model = CNNMnist()
elif dataset == 'cifar':
self.global_model = CNNCifar()
elif model == 'mlp':
self.global_model = MLP(dim_in=img_size, dim_hidden=64, dim_out=10)
elif model == 'densenet':
self.global_model = torch.hub.load('pytorch/vision:v0.5.0',
'densenet121',
pretrained=False)
elif model == 'vgg19':
self.global_model = torch.hub.load('pytorch/vision:v0.5.0',
'vgg19',
pretrained=False)
else:
exit('Error: unrecognized model')
self.ready_client_sids = set()
self.app = Flask(__name__)
self.socketio = SocketIO(self.app)
self.host = host
self.port = port
self.model_id = str(uuid.uuid4())
self.current_round = -1 # -1 for not yet started
self.current_round_client_updates = []
self.eval_client_updates = []
self.register_handles()
def build_model(args, in_feats, n_hidden, n_classes, device, n_layers=1):
if args.model == 'gcn_cv_sc':
infer_device = torch.device("cpu") # for sampling
train_model = GCNSampling(in_feats, n_hidden, n_classes, 2, F.relu,
args.dropout).to(device)
infer_model = GCNInfer(in_feats, args.n_hidden, n_classes, 2, F.relu)
model = (train_model, infer_model)
elif args.model == 'gs-mean':
model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout, 'mean').to(device)
elif args.model == 'mlp':
model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif args.model == 'mostfrequent':
model = MostFrequentClass()
# elif args.model == 'egcn':
# if n_layers != 2:
# print("Warning, EGCN doesn't respect n_layers")
# egcn_args = egcn_utils.Namespace({'feats_per_node': in_feats,
# 'layer_1_feats': n_hidden,
# 'layer_2_feats': n_classes})
# model = EGCN(egcn_args, torch.nn.RReLU(), device=device, skipfeats=False)
elif args.model == 'gat':
print("Warning, GAT doesn't respect n_layers")
heads = [8, args.gat_out_heads] # Fixed head config
# Div num_hidden by heads for same capacity
n_hidden_per_head = int(n_hidden / heads[0])
assert n_hidden_per_head * heads[
0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}"
model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads, F.elu,
0.6, 0.6, 0.2, False).to(device)
else:
raise NotImplementedError("Model not implemented")
return model
def make_data_and_model(N, D, L, seed=1):
"""
# N: Number of samples
# D: Dimension of input and of each Layer
# L: Number of hidden layers
"""
torch.manual_seed(seed)
hidden_sizes = list(D for l in range(L))
X = torch.Tensor(torch.randn(N, D))
y = torch.Tensor(torch.round(torch.rand(N))).view(-1, )
model = MLP(input_size=D, hidden_sizes=hidden_sizes)
model.train(True)
return X, y, model
def get_model(in_feats, label_in_feats, n_classes, stage, args, subset_list=None):
num_hops = args.K + 1
use_labels = args.use_labels and ((not args.inductive) or stage > 0)
if args.model == "sagn":
base_model = SAGN(in_feats, args.num_hidden, n_classes, num_hops,
args.mlp_layer, args.num_heads,
weight_style=args.weight_style,
dropout=args.dropout,
input_drop=args.input_drop,
attn_drop=args.attn_drop,
zero_inits=args.zero_inits,
position_emb=args.position_emb,
focal=args.focal)
if args.model == "mlp":
base_model = MLP(in_feats, args.num_hidden, n_classes,
args.mlp_layer,
args.dropout,
residual=False,
input_drop=args.input_drop)
if args.model == "plain_sagn":
base_model = PlainSAGN(in_feats, args.num_hidden, n_classes,
args.mlp_layer, args.num_heads,
dropout=args.dropout,
input_drop=args.input_drop,
attn_drop=args.attn_drop,
zero_inits=args.zero_inits)
if args.model == "sign":
base_model = SIGN(in_feats, args.num_hidden, n_classes, num_hops,
args.mlp_layer,
dropout=args.dropout,
input_drop=args.input_drop)
if args.avoid_features:
base_model = None
if args.dataset == "ogbn-mag":
assert base_model is not None
base_model = NARS(in_feats, num_hops, base_model, subset_list)
if use_labels:
label_model = GroupMLP(label_in_feats,
args.num_hidden,
n_classes,
args.num_heads,
args.label_mlp_layer,
args.label_drop,
residual=args.label_residual,)
else:
label_model = None
model = SLEModel(base_model, label_model)
return model
def __init__(self, mlp_kwargs, n1_kwargs, n2_kwargs, n3_kwargs=None):
super(Medusa, self).__init__()
self.name = 'medusa'
self.n1 = CNN1D(**n1_kwargs)
self.n2 = MobileNetV2(**n2_kwargs)
self.n3 = None
if n3_kwargs:
self.n3 = CNN1D(**n3_kwargs)
self.mlp = MLP(**mlp_kwargs)
def get_model(args):
if args.model == 'mlp':
model = MLP(num_classes=args.n_classes)
elif args.model == 'resnet':
model = ResNet(20, num_classes=args.n_classes)
elif args.model == 'densenet':
model = DenseNet(40, num_classes=args.n_classes)
return model
def build_model(args,
in_feats,
n_hidden,
n_classes,
device,
n_layers=1,
backend='geometric',
edge_index=None,
num_nodes=None):
if backend == 'geometric':
if args.model == 'gs-mean':
model = geo.GraphSAGE(in_feats, n_hidden, n_classes, n_layers,
F.relu, args.dropout).to(device)
elif args.model == "gcn":
model = geo.GCN(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif args.model == "gat":
print("Warning, GAT doesn't respect n_layers")
heads = [8, args.gat_out_heads] # Fixed head config
n_hidden_per_head = int(n_hidden / heads[0])
model = geo.GAT(in_feats, n_hidden_per_head, n_classes, F.relu,
args.dropout, 0.6, heads).to(device)
elif args.model == "mlp":
model = geo.MLP(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif args.model == "jknet":
model = geo.JKNet(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif args.model == "sgnet":
model = SGNet(in_channels=in_feats,
out_channels=n_classes,
K=n_layers).to(device)
else:
raise NotImplementedError
else:
if args.model == 'gs-mean':
model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout, 'mean').to(device)
elif args.model == 'mlp':
model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif args.model == 'mostfrequent':
model = MostFrequentClass()
elif args.model == 'gat':
print("Warning, GAT doesn't respect n_layers")
heads = [8, args.gat_out_heads] # Fixed head config
# Div num_hidden by heads for same capacity
n_hidden_per_head = int(n_hidden / heads[0])
assert n_hidden_per_head * heads[
0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}"
model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads,
F.elu, 0.6, 0.6, 0.2, False).to(device)
else:
raise NotImplementedError("Model not implemented")
return model
def GetModel(op, client_id, global_round):
# initial
global stop_round
if op == 0:
if model == 'cnn':
if dataset == 'mnist' or dataset == 'fmnist':
global_model = CNNMnist()
elif dataset == 'cifar':
global_model = CNNCifar()
elif model == 'mlp':
global_model = MLP(dim_in=img_size, dim_hidden=64, dim_out=10)
elif model == 'densenet':
global_model = torch.hub.load('pytorch/vision:v0.5.0',
'densenet121',
pretrained=False)
elif model == 'vgg19':
global_model = torch.hub.load('pytorch/vision:v0.5.0',
'vgg19',
pretrained=False)
else:
exit('Error: unrecognized model')
encoing_string = base64.b64encode(pickle.dumps(global_model)).decode()
clients_arr.remove(client_id)
return encoing_string
else:
f_path_global = 'Models/global_model' + str(global_round) + '.pkl'
if client_id == 1:
while 1:
filenums, files = GetFileNum(global_round)
if filenums == client_nums:
weights = []
for f in files:
f_path = "Models/LocalModels/" + str(
global_round) + '/' + f
weights.append(torch.load(f_path))
new_global_model, cvg_flag = Average_weight(weights)
if cvg_flag and stop_round is None:
stop_round = global_round
if global_round == total_global_round:
print(global_loss_per_epoch, stop_round)
print(f_path_global)
save(new_global_model, f_path_global)
#torch.save(new_global_model, f_path_global)
with open(f_path_global, 'rb') as file:
encoing_string = base64.b64encode(file.read())
clients_arr.remove(client_id)
return encoing_string
time.sleep(2)
else:
while 1:
if os.path.isfile(f_path_global):
with open(f_path_global, 'rb') as file:
encoing_string = base64.b64encode(file.read())
clients_arr.remove(client_id)
return encoing_string
time.sleep(2)
def main():
save_dir = './results/'
models = [
MLP(
num_layers=NUM_LAYERS,
in_dim=2,
hidden_dim=HIDDEN_DIM,
out_dim=1,
activation='relu6',
),
MLP(
num_layers=NUM_LAYERS,
in_dim=2,
hidden_dim=HIDDEN_DIM,
out_dim=1,
activation='none',
),
NAC(
num_layers=NUM_LAYERS,
in_dim=2,
hidden_dim=HIDDEN_DIM,
out_dim=1,
),
NALU(
num_layers=NUM_LAYERS,
in_dim=2,
hidden_dim=HIDDEN_DIM,
out_dim=1
),
]
results = {}
for fn_str, fn in ARITHMETIC_FUNCTIONS.items():
print('[*] Testing function: {}'.format(fn_str))
results[fn_str] = []
# dataset
num_train=500, num_test=50,
dim=100, num_sum=5, fn=fn,
support=RANGE,
)
def MLP_forecasting(dataset,
inputDim,
lr=1e-3,
hiddenNum=50,
outputDim=1,
epoch=20,
batchSize=30,
plot_flag=False):
# normalize time series
# dataset = dataset.reshape(-1, 1)
scaler = MinMaxScaler(feature_range=(0.0, 1.0)).fit(dataset)
dataset = scaler.transform(dataset)
# divide the series into training/testing samples
# NOTE: Not RNN format
train, test = util.divideTrainTest(dataset)
trainX, trainY = util.createSamples(train, inputDim, RNN=False)
testX, testY = util.createSamples(test, inputDim, RNN=False)
print("trainX shape is", trainX.shape)
print("trainY shape is", trainY.shape)
print("testX shape is", testX.shape)
print("testY shape is", testY.shape)
# buil model and train
MLP_model = MLP.MLP_Model(inputDim, hiddenNum, outputDim, lr)
t1 = time.time()
MLP_model.train(trainX, trainY, epoch, batchSize)
t2 = time.time() - t1
print("train time is", t2)
# forecasting
trainPred = MLP_model.predict(trainX)
testPred = MLP_model.predict(testX)
# reverse the time series
trainPred = scaler.inverse_transform(trainPred)
trainY = scaler.inverse_transform(trainY)
testPred = scaler.inverse_transform(testPred)
testY = scaler.inverse_transform(testY)
# evaluate
MAE = eval.calcMAE(testY, testPred)
print("test MAE", MAE)
MRSE = eval.calcRMSE(testY, testPred)
print("test RMSE", MRSE)
SMAPE = eval.calcSMAPE(testY, testPred)
print("test SMAPE", SMAPE)
if plot_flag:
util.plot(trainPred, trainY, testPred, testY)
return trainPred, testPred, MAE, MRSE, SMAPE
def load_model(num_classes_to_predict):
# vgg = SimpleNet(num_classes_to_predict)
low_n_units = [32 * 32 * 3, 512, num_classes_to_predict]
high_n_units = [32 * 32 * 3, 2048, 512, 256, num_classes_to_predict]
lowest_n_units = [32 * 32 * 3, 128, num_classes_to_predict]
low_n_units_mnist = [28 * 28, 256, num_classes_to_predict]
high_n_units_mnist = [28 * 28, 512, 256, 128, 64, num_classes_to_predict]
model = MLP(low_n_units)
return model
def run():
# basics
model = MLP([HIDDENS, HIDDENS, 10], config=config).to(DEVICE)
# model = ResNet18(100, 20, config=config).to(DEVICE)
tasks = get_rotated_mnist_tasks(NUM_TASKS, shuffle=True, batch_size=BATCH_SIZE)
# optimizer = torch.optim.SGD(model.parameters(), lr=config['lr'], momentum=config['momentum'])
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=config['gamma'])
criterion = nn.CrossEntropyLoss().to(DEVICE)
# hooks
setup_experiment()
# main loop
time = 0
for current_task_id in range(1, NUM_TASKS+1):
print("========== TASK {} / {} ============".format(current_task_id, NUM_TASKS))
train_loader = tasks[current_task_id]['train']
# task_lr = config['lr']*(config['gamma']**(current_task_id-1))
for epoch in range(1, EPOCHS+1):
# train and save
lr = max(config['lr']* config['gamma']**(current_task_id), config['lrlb'])
print(lr)
optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=config['momentum'])
train_single_epoch(model, optimizer, train_loader, criterion, current_task_id)
time += 1
# evaluate on all tasks up to now
for prev_task_id in range(1, current_task_id+1):
model = model.to(DEVICE)
val_loader = tasks[prev_task_id]['test']
if epoch == EPOCHS:
metrics = eval_single_epoch(model, val_loader, criterion, prev_task_id)
log_metrics(metrics, time, prev_task_id)
save_checkpoint(model, time)
if prev_task_id == current_task_id:
log_hessian(model, val_loader, time, prev_task_id)
# save_checkpoint(model, time)
# scheduler.step()
end_experiment()
def test_cifar(data_path):
print 'loading data'
dl = CIFAR10DataLoader(cifar_file_path=data_path,
load_dtype='float32')
dp = LabeledMemoryDataProvider(data_loader=dl,
batch_size=batch_size,
max_gpu_train_samples=max_gpu_samples,
max_gpu_valid_samples=max_gpu_samples,
is_test=False,
epochwise_shuffle=False,
nvalid_samples=5000)
opt = optimizers.SGD_Momentum_Optimizer()
mlp = MLP(batch_size = batch_size, seed=seed,
network_structure=mlp_ns, network_cost=mlp_cost)
mlp.fit(data_provider=dp,
optimizer=opt,
train_epoch=training_epochs,
early_stop=10,
dump_freq=(50000//100+1)*10,
train_mean=dp.get_data_stats(),
batch_mean_subtraction=True)
mlp.save('alex_cifar_26_model.joblib')
# mlp.load('alex_cifar_26_model.joblib')
#
f = open(os.path.join(data_path,'test_batch'), 'rb')
data = cPickle.load(f)
f.close()
test_set = data['data']
test_y = data['labels']
test_set = numpy.asarray(test_set, dtype='float32')
tm = dp.get_data_stats()
test_set -= tm
pred = mlp.predict_from_memory_data(test_set)
print 'Error: ', numpy.mean(pred!=test_y)
dp = LabeledMemoryDataProvider(data_loader=dl,
batch_size=batch_size,
max_gpu_train_samples=max_gpu_samples,
max_gpu_valid_samples=max_gpu_samples,
is_test=True,
epochwise_shuffle=False,
nvalid_samples=0)
pred = mlp.predict_from_data_provider(dp, train_mean=tm, batch_mean_subtraction=True)
print 'Error: ', numpy.mean(pred!=test_y)
def test_finetune():
dl = MNISTDataLoader('/data/Research/datasets/mnist/mnist.pkl')
dp = LabeledMemoryDataProvider(data_loader=dl,
batch_size=batch_size,
max_gpu_train_samples=max_gpu_samples,
max_gpu_valid_samples=max_gpu_samples,
is_test=False,
epochwise_shuffle=False,
nvalid_samples=10000)
opt = optimizers.SGD_Rms_Optimizer()
init_params = hickle.load('testmodel_ae.joblib')
mlp = MLP(batch_size = batch_size, seed=seed,
network_structure=mlp_ns, network_cost=mlp_cost,
init_params=init_params)
mlp.fit(data_provider=dp,
optimizer=opt,
train_epoch=finetune_epochs,
early_stop=False,
dump_freq=(50000//100+1)*10)
mlp.save('mnist_mlp_model.joblib')
f = open('/data/Research/datasets/mnist/mnist.pkl', 'rb')
_, _, test_set = cPickle.load(f)
f.close()
pred = mlp.predict_from_memory_data(test_set[0])
print 'Error: ', numpy.mean(pred!=test_set[1])
dp = LabeledMemoryDataProvider(data_loader=dl,
batch_size=batch_size,
max_gpu_train_samples=max_gpu_samples,
max_gpu_valid_samples=max_gpu_samples,
is_test=True,
epochwise_shuffle=False,
nvalid_samples=0)
pred = mlp.predict_from_data_provider(dp)
print 'Error: ', numpy.mean(pred!=test_set[1])
