def restore_model(file_path: str): info = torch.load(file_path) arch: str = info['hparams'].arch batch_norm = info['hparams'].batch_norm dataset: str = info['hparams'].dataset hidden_layers: int = info['hparams'].hidden_layers hidden_size: int = info['hparams'].hidden_size if arch == 'mlp' and dataset == 'mnist': model: nn.Module = MLP(input_size=784, hidden_size=hidden_size, num_hidden_layers=hidden_layers, batch_norm=batch_norm) elif arch == 'alexnet': model: nn.Module = AlexNet() else: model: nn.Module = MLP(hidden_size=hidden_size, num_hidden_layers=hidden_layers, batch_norm=batch_norm) model.load_state_dict(info['model_state_dict']) lr = info['hparams'].lr momentum = info['hparams'].momentum weight_decay = info['hparams'].weight_decay optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay) optimizer.load_state_dict(info['optimizer_state_dict']) # model.eval() return model, optimizer, info['hparams']
def configure(**kwargs): """Configure critic.""" observation_space = kwargs['observation_space'] action_space = kwargs['action_space'] assert isinstance(observation_space, Box) nb_state_feats = observation_space.shape[-1] net_dict = dict(nb_layers=kwargs['nb_layers'], hidden_size=kwargs['hidden_size']) val_function = MLP(nb_inputs=nb_state_feats, nb_outputs=1, **net_dict) if isinstance(action_space, Discrete): nb_actions = action_space.n adv_function = MLP(nb_inputs=nb_state_feats, nb_outputs=nb_actions, **net_dict) elif isinstance(action_space, Box): nb_actions = action_space.shape[-1] adv_function = ContinuousAdvantageMLP( nb_outputs=1, nb_state_feats=nb_state_feats, nb_actions=nb_actions, **net_dict) if kwargs['normalize']: val_function = NormalizedMLP(val_function) adv_function = NormalizedMLP(adv_function) return AdvantageCritic(kwargs['dt'], kwargs['gamma'], kwargs['lr'], kwargs['tau'], kwargs['optimizer'], val_function, adv_function)
def configure_model(self): """ :return: """ arch: str = self.hparams.arch batch_norm = self.hparams.batch_norm dataset: str = self.hparams.dataset hidden_layers: int = self.hparams.hidden_layers hidden_size: int = self.hparams.hidden_size if arch == 'mlp': if dataset == 'mnist': return MLP(input_size=784, hidden_size=hidden_size, num_hidden_layers=hidden_layers, batch_norm=batch_norm) elif dataset == 'cifar10': return MLP(hidden_size=hidden_size, num_hidden_layers=hidden_layers, batch_norm=batch_norm) else: raise ValueError('invalid dataset specification!') elif arch == 'alexnet': return AlexNet() elif arch == 'vgg11': return VGG(vgg_name='VGG11') elif arch == 'vgg13': return VGG(vgg_name='VGG13') elif arch == 'resnet18': return ResNet18() elif arch == 'resnet34': return ResNet34() else: raise ValueError('Unsupported model!')
def estimate_divergences(model_hyperparams, lr, criterion, device, batch_size, n_epochs, is_wasserstein): input_size, h1_size, h2_size, out_size, out_sigmoid = model_hyperparams p_dist = iter(samplers.distribution1(x=0, batch_size=batch_size)) # train a model for each value of phi phi_list = np.linspace(-1, 1, 21) jsd_list = [] for phi in phi_list: # create model and optimizer model = MLP(input_size, h1_size, h2_size, out_size, out_sigmoid).to(device) print(model) optimizer = torch.optim.SGD(model.parameters(), lr=lr) q_dist = iter(samplers.distribution1(x=phi, batch_size=batch_size)) losses = train(model, p_dist, q_dist, optimizer, criterion, device, n_epochs, is_wasserstein) # visualise loss. # plt.figure() # plt.plot(losses) # plt.title('phi = {}'.format(phi)) # plt.show() divergence_estimate = -1 * losses[-1] print('At phi = {}, divergence estimate = {}'.format(phi, divergence_estimate)) jsd_list.append(divergence_estimate) plt.figure() plt.plot(phi_list, jsd_list, 'o') plt.xlabel('$\phi$', fontsize=14) plt.ylabel('Jensen-Shannon Divergence', fontsize=14) plt.show()
def __init__(self, args, num_users, num_items): BaseModel.__init__(self, args, num_users, num_items) self.layers = eval(args.layers) self.lambda_layers = eval(args.reg_layers) self.num_factors = args.num_factors self.model_GMF = GMF(args, num_users, num_items) self.model_MLP = MLP(args, num_users, num_items)
def main(): save_dir = './imgs/' TRAIN_RANGE[-1] += 1 TEST_RANGE[-1] += 1 # datasets train_data = torch.arange(*TRAIN_RANGE).unsqueeze_(1).float() test_data = torch.arange(*TEST_RANGE).unsqueeze_(1).float() # train all_mses =[] for non_lin in NON_LINEARITIES: print("Working with {}...".format(non_lin)) mses = [] for i in range(100): net = MLP(4, 1, 8, 1, non_lin) optim = torch.optim.RMSprop(net.parameters(), lr=LEARNING_RATE) train(net, optim, train_data, NUM_ITERS) mses.append(test(net, test_data)) all_mses.append(torch.cat(mses, dim=1).mean(dim=1)) all_mses = [x.numpy().flatten() for x in all_mses] # plot fig, ax = plt.subplots(figsize=(8, 7)) x_axis = np.arange(-20, 21) for i, non_lin in enumerate(NON_LINEARITIES): ax.plot(x_axis, all_mses[i], label=non_lin) plt.grid() plt.legend(loc='best') plt.ylabel('Mean Absolute Error') plt.savefig(save_dir + 'extrapolation.png', format='png', dpi=300) plt.show()
def __init__(self, dataset, pretrained_net=None, x_dim=1, y_dim=1, h_dim=32, lr=0.1, bs=8, device='cpu', acq_scheme='rand'): super(PseudoBO, self).__init__() self.R = dataset.inquiry self.device = device self.inn_lr = lr self.bs = bs self.acq_scheme = acq_scheme self.net = MLP(x_dim, y_dim, h_dim) if pretrained_net is None else pretrained_net self.net.to(device) # init w_list self.inn_grad_step() # init D_0 self.D = [] s = dataset.data[0].to(self.device) self.D.append(s)
def train_model(train_tcrs, train_pathologies, test_tcrs, test_pathologies, device, args, params): """ Train and evaluate the model """ losses = [] # We use Cross-Entropy loss loss_function = nn.CrossEntropyLoss() # Set model with relevant parameters model = MLP(params['enc_dim'], params['out_dim'], device) # Move to GPU model.to(device) # We use Adam optimizer optimizer = optim.Adam(model.parameters(), lr=params['lr'], weight_decay=params['wd']) # Train several epochs for epoch in range(params['epochs']): print('epoch:', epoch + 1) # Train model and get loss loss = train_epoch(train_tcrs, train_pathologies, model, loss_function, optimizer, device) losses.append(loss) # evaluate acc = evaluate(model, train_tcrs, train_pathologies, device) print('train accuracy:', acc) acc = evaluate(model, test_tcrs, test_pathologies, device) print('test accuracy:', acc) return model
def main(): test_seen_loader = torch.utils.data.DataLoader(AttributeDataset( args.data_dir, args.dataset, features_path=args.gan_path, mode='test_seen', generalized=True, normalize=args.normalize, sentences=args.sentences), batch_size=args.batch_size, shuffle=False) test_unseen_loader = torch.utils.data.DataLoader( AttributeDataset(args.data_dir, args.dataset, features_path=args.gan_path, mode='test_unseen', generalized=True, normalize=args.normalize, sentences=args.sentences), batch_size=args.batch_size, shuffle=False) # instanciate the models if args.mlp: mlp = MLP(args.dim_input, [args.nhidden * 2], args.nhidden) else: mlp = LinearProjection(args.dim_input, args.nhidden) embed = LinearProjection(args.nhidden, args.dim_embed) if args.sentences: cam_key = 'sentences' else: cam_key = 'emb' if args.gan_path is not None: cam_key = 'full_' + cam_key cam = torch.from_numpy(test_seen_loader.dataset.data[cam_key].T) proxies = ProxyNet(args.n_classes, args.dim_embed, proxies=cam) model = Base(mlp, embed, proxies) criterion = ProxyLoss(temperature=args.temp) if args.cuda: mlp.cuda() embed.cuda() model.cuda() proxies.cuda() # loading checkpoint = torch.load(args.model_path) model.load_state_dict(checkpoint['state_dict']) txt = ("=> loaded checkpoint '{}' (epoch {})".format( args.model_path, checkpoint['epoch'])) print(txt) compute_scores(test_seen_loader, test_unseen_loader, model, criterion)
def main(): __model = MLP(in_dim=400, hidden_dim=HIDDEN_DIM, out_dim=800) print("\tTraining {}...".format(__model.__str__().split("(")[0])) optim = torch.optim.Adam(__model.parameters(), lr=LEARNING_RATE) train(__model, optim, X_train, y_train, 128) mse = test(__model, X_test, y_test).mean().item() print(mse)
def check_model(self, X_train, X_val, y_train, y_val, X_test, y_test, raw_seq): """ Funtion used to navigate to the specific model. The is defined when initialising the class. Reads the self.model_type Each statement does the following: - Calls function to format data for the model - Calls funtion to train the model - Calls funtion to plot the MSE graph - Calls funtion to test the model - Returns the accuarcy as R2 score""" if self.model_type == 'CNN': X_train, X_val, y_train, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4 = CNN.data_format( X_train, X_val, y_train) history = CNN.CNN_train_model(self, X_train, X_val, y_train, y_val, self.verbose, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4) Models.plotting(history) yhat = CNN.CNN_test_model(self, X_test, self.verbose, y_test) Models.accuracy(self, yhat, y_test, X_test, self.model_type) if self.model_type == 'MLP': X_train, X_val, y_train, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4 = MLP.data_format( X_train, X_val, y_train) history = MLP.MLP_train_model(self, X_train, X_val, y_train, y_val, self.verbose, n_input, n_output, ytrain1, ytrain2, ytrain3, ytrain4) # Models.plotting(history) yhat, final_cols = MLP.MLP_test_model(X_test, self.verbose, y_test) Models.accuracy(self, yhat, y_test, final_cols, self.model_type) if self.model_type == 'KNN': X_train, X_val, y_train, X_test = KNN.data_format( X_train, X_val, y_train, X_test) yhat, final_cols = KNN.KNN_train_model(self, X_train, X_val, y_train, y_val, X_test, y_test, raw_seq) Models.accuracy(self, yhat, y_test, final_cols, self.model_type) if self.model_type == 'LSTM': history, model = LSTMs.LSTM_train_model(self, X_train, X_val, y_train, y_val, self.verbose) Models.plotting(history) yhat = LSTMs.LSTM_test_model(X_test, model, self.verbose, y_test) Models.accuracy(self, yhat, y_test, X_test, self.model_type) if self.model_type == 'BASELINE': n_input, X_train, n_output = BaseLine.data_format(X_train, y_train) model = BaseLine.baseline_train(self, X_train, y_train, n_input, n_output) yhat, final_cols = BaseLine.baseline_test(X_test, n_input, model) Models.accuracy(self, yhat, y_test, final_cols, self.model_type)
def main(args): print('[*] Arguments: %s' % args) print('[*] Read MNIST...') num_test_images = args.num_test_images images, labels = load_mnist('test', path='./data', max_ind=num_test_images) images, labels = images[:num_test_images, :, :], labels[:num_test_images] images = images.astype(np.float32) images = images / 255. print('[*] The shape of image: %s' % str(images.shape)) print('[*] Load the network...') if args.network == 'mlp': # Lab 2 if args.quantized: print('[!] MLP does not support quantization') return model_path = os.path.join('./pretrained_weights', 'mlp_iter_10000.caffemodel') net = MLP(model_path, args) elif args.network == 'cnn': if args.quantized: # Lab 14 model_path = os.path.join('./pretrained_weights', 'quantized_cnn_weights.txt') else: # Lab 11 model_path = os.path.join('./pretrained_weights', 'cnn_weights.txt') net = CNN(model_path, args) else: raise print('[*] Run tests...') test_images = [images[i, :, :].copy() for i in xrange(num_test_images)] n_correct = 0 start_time = time.time() for i in xrange(num_test_images): X = test_images[i] X = X.reshape((28 * 28)) # 28x28->784 logit = net.inference(X) prediction = np.argmax(logit) label = labels[i, ] n_correct += (label == prediction) print('[*] Statistics...') model_stats = { 'total_time': time.time() - start_time, 'total_image': num_test_images, 'accuracy': float(n_correct) / num_test_images, 'avg_num_call': net.total_num_call[0] / num_test_images, 'm_size': net.m_size, 'v_size': net.v_size, } pp.pprint(model_stats)
def main(): # MNIST dataset train_dataset = torchvision.datasets.MNIST( root='/Datadisk/jmlu/Pytorch_Data/', train=True, transform=transforms.ToTensor(), download=False) test_dataset = torchvision.datasets.MNIST( root='/Datadisk/jmlu/Pytorch_Data/', train=False, transform=transforms.ToTensor()) # Data loader train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False) model = MLP(input_size, hidden_size, num_classes).to(device) print(model) # Loss and optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) if Baseline: print("Baselie Training ----------------") train(model, train_loader, num_epochs, device, optimizer, criterion) test(model, test_loader, device) # Save the model checkpoint torch.save(model.state_dict(), 'model_base.ckpt') exit() if Prune: print("Model Pruning -------------------") model.load_state_dict(torch.load('model_base.ckpt')) model_prune(model) print('\nTesting After Prune: ') test(model, test_loader, device) torch.save(model.state_dict(), 'model_pruned.ckpt') exit() if Retrain: print("Model Retrain -------------------") model.load_state_dict(torch.load('model_pruned.ckpt')) model_finetune(model, train_loader, num_epochs, device, optimizer, criterion) test(model, test_loader, device) torch.save(model.state_dict(), 'model_finetuned.ckpt') exit()
def __init__(self, args): self.args = args self.model = MLP(args.nin, args.nh, args.nout, args.do) self.model.to(args.d) print("model params {}".format(count_parameters(self.model))) log_path = "logs/gmm" if os.path.exists(log_path) and os.path.isdir(log_path): shutil.rmtree(log_path) self.writer = SummaryWriter(log_path) nc = 2 if args.dataset == "toy" else 3 self.best_loss = np.inf self.softmax = torch.nn.Softmax(dim=1) self.ce = torch.nn.CrossEntropyLoss()
def train_minibatch(): generator = MiniBatchGenerator(861, x_mapper, y_mapper) generator.split_train_test() """ print('load train mini-batch') while True: X, y = generator.load_next_train_batch(10) if X is None: break print(y) X_to_train, y_to_train = preprocess(X, y) print(X_to_train.shape) print(y_to_train.shape) break sess, y_predict, x_train, y_train = models.train_nn(NUM_CATEGORIES, X_to_train, y_to_train, layers=(256, 256), iterations=1000) print('load test mini-batch') while True: X, y = generator.load_next_test_batch(10) if X is None: break X_to_test, y_to_test = preprocess(X, y) correct_prediction = tf.equal(tf.argmax(y_predict, 1), tf.argmax(y_train, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print(sess.run(accuracy, feed_dict={x_train: X_to_test, y_train: y_to_test})) correct_prediction = tf.equal(tf.argmax(y_predict, 1), tf.argmax(y_train, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #print(sess.run(accuracy, feed_dict={x_train: X_to_train, y_train: y_to_train})) print(sess.run(accuracy, feed_dict={x_train: X_to_test, y_train: y_to_test})) """ model = MLP(NUM_CATEGORIES, min_pc, layers=(256, 256, 256), learning_rate=0.0005) model.start_session() X_test, y_test = generator.load_next_test_batch(1000) X_test, y_test = preprocess(X_test, y_test) #sess = None for iteration in range(1000): generator.reset() while True: X_train, y_train = generator.load_next_train_batch(10) if X_train is None: break X_train, y_train = preprocess(X_train, y_train) model.train(X_train, y_train) if iteration % 100 == 0: print('loss = ', model.calculate_loss()) # print('train acc = ', model.calculate_accuracy(X_train, y_train)) print('test acc = ', model.calculate_accuracy(X_test, y_test)) print("\n---\n")
def mc_mlp_val(x_train, y_train, size_input, hidden_layers, nodes_hidden_layers, size_output=2, runs=1000, n_samples=5): max_acc = 0.0 max_conf = [] es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=0, restore_best_weights=True) mlp_cache = {} for _ in tqdm(range(runs)): # generate random mlp conf = gen_random_mlp(size_input, hidden_layers, nodes_hidden_layers, size_output) if conf not in mlp_cache: acc = 0.0 for _ in range(n_samples): model = MLP(conf, use_bias_input=False) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=50, batch_size=300, validation_split=0.2, callbacks=[es], verbose=0) acc += history.history['val_accuracy'][-1] acc /= n_samples mlp_cache[conf] = acc if acc > max_acc: max_acc = acc max_conf = conf return max_conf, max_acc
def set_up_predictor(fp_out_dim, net_hidden_dims, class_num, sim_method='mlp', symmetric=None): sim_method_dict = { 'mlp': 'multi-layered perceptron', 'ntn': 'bilinear transform', 'symmlp': 'symmetric perceptron', 'hole': 'holographic embedding', 'dist-mult': 'dist-mult', } logging.info('Link Prediction: {}'.format( sim_method_dict.get(sim_method, None))) lp = None if sim_method == 'mlp': lp = MLP(out_dim=class_num, hidden_dims=net_hidden_dims) elif sim_method == 'ntn': ntn_out_dim = 8 lp = NTN(left_dim=fp_out_dim, right_dim=fp_out_dim, out_dim=class_num, ntn_out_dim=ntn_out_dim, hidden_dims=net_hidden_dims) elif sim_method == 'symmlp': lp = MLP(out_dim=class_num, hidden_dims=net_hidden_dims) elif sim_method == 'hole': lp = HolE(out_dim=class_num, hidden_dims=net_hidden_dims) elif sim_method == 'dist-mult': dm_out_dim = 8 lp = DistMult(left_dim=fp_out_dim, right_dim=fp_out_dim, out_dim=class_num, dm_out_dim=dm_out_dim, hidden_dims=net_hidden_dims) else: raise ValueError( '[ERROR] Invalid link prediction model: {}'.format(sim_method)) predictor = DDIPredictor(lp, symmetric=symmetric) return predictor
def main(args): exp_info = exp_config.Experiment(args.dataset) paths = exp_info.paths args.paths = paths args.metadata = exp_info.metadata np.random.seed(args.seed) torch.manual_seed(args.seed) args.batch_size = 1 feature_size, train_loader, val_loader, test_loader, all_loader = exp_info.get_dataset( args, save=True) label_num = exp_info.get_label_num(args) hidden_size = 256 hidden_layers = 2 if args.model == 'lstm': parsing_model = lstm_model.BiLSTM(feature_size, hidden_size, hidden_layers, label_num) else: parsing_model = mlp_model.MLP(feature_size, hidden_size, label_num) parsing_model = torch.nn.DataParallel(parsing_model) prev = args.subsample args.subsample = 1 args.save_path = os.path.join(paths.inter_root, 'likelihood', args.task, args.model) args.resume = os.path.join( paths.checkpoint_root, 'detection_{}_{}_e{}_lr{}_b{}_lrd{}_s{}_do{}'.format( args.task, args.model, args.epochs, args.lr, args.using_batch_size, args.lr_decay, 1 if not args.subsample else args.subsample, args.dropout_rate)) args.subsample = prev logutils.load_checkpoint(args, parsing_model) validate(test_loader, parsing_model, args=args)
def __init__(self, host, port): if model == 'cnn': if dataset == 'mnist' or dataset == 'fmnist': self.global_model = CNNMnist() elif dataset == 'cifar': self.global_model = CNNCifar() elif model == 'mlp': self.global_model = MLP(dim_in=img_size, dim_hidden=64, dim_out=10) elif model == 'densenet': self.global_model = torch.hub.load('pytorch/vision:v0.5.0', 'densenet121', pretrained=False) elif model == 'vgg19': self.global_model = torch.hub.load('pytorch/vision:v0.5.0', 'vgg19', pretrained=False) else: exit('Error: unrecognized model') self.ready_client_sids = set() self.app = Flask(__name__) self.socketio = SocketIO(self.app) self.host = host self.port = port self.model_id = str(uuid.uuid4()) self.current_round = -1 # -1 for not yet started self.current_round_client_updates = [] self.eval_client_updates = [] self.register_handles()
def build_model(args, in_feats, n_hidden, n_classes, device, n_layers=1): if args.model == 'gcn_cv_sc': infer_device = torch.device("cpu") # for sampling train_model = GCNSampling(in_feats, n_hidden, n_classes, 2, F.relu, args.dropout).to(device) infer_model = GCNInfer(in_feats, args.n_hidden, n_classes, 2, F.relu) model = (train_model, infer_model) elif args.model == 'gs-mean': model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, 'mean').to(device) elif args.model == 'mlp': model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == 'mostfrequent': model = MostFrequentClass() # elif args.model == 'egcn': # if n_layers != 2: # print("Warning, EGCN doesn't respect n_layers") # egcn_args = egcn_utils.Namespace({'feats_per_node': in_feats, # 'layer_1_feats': n_hidden, # 'layer_2_feats': n_classes}) # model = EGCN(egcn_args, torch.nn.RReLU(), device=device, skipfeats=False) elif args.model == 'gat': print("Warning, GAT doesn't respect n_layers") heads = [8, args.gat_out_heads] # Fixed head config # Div num_hidden by heads for same capacity n_hidden_per_head = int(n_hidden / heads[0]) assert n_hidden_per_head * heads[ 0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}" model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads, F.elu, 0.6, 0.6, 0.2, False).to(device) else: raise NotImplementedError("Model not implemented") return model
def make_data_and_model(N, D, L, seed=1): """ # N: Number of samples # D: Dimension of input and of each Layer # L: Number of hidden layers """ torch.manual_seed(seed) hidden_sizes = list(D for l in range(L)) X = torch.Tensor(torch.randn(N, D)) y = torch.Tensor(torch.round(torch.rand(N))).view(-1, ) model = MLP(input_size=D, hidden_sizes=hidden_sizes) model.train(True) return X, y, model
def get_model(in_feats, label_in_feats, n_classes, stage, args, subset_list=None): num_hops = args.K + 1 use_labels = args.use_labels and ((not args.inductive) or stage > 0) if args.model == "sagn": base_model = SAGN(in_feats, args.num_hidden, n_classes, num_hops, args.mlp_layer, args.num_heads, weight_style=args.weight_style, dropout=args.dropout, input_drop=args.input_drop, attn_drop=args.attn_drop, zero_inits=args.zero_inits, position_emb=args.position_emb, focal=args.focal) if args.model == "mlp": base_model = MLP(in_feats, args.num_hidden, n_classes, args.mlp_layer, args.dropout, residual=False, input_drop=args.input_drop) if args.model == "plain_sagn": base_model = PlainSAGN(in_feats, args.num_hidden, n_classes, args.mlp_layer, args.num_heads, dropout=args.dropout, input_drop=args.input_drop, attn_drop=args.attn_drop, zero_inits=args.zero_inits) if args.model == "sign": base_model = SIGN(in_feats, args.num_hidden, n_classes, num_hops, args.mlp_layer, dropout=args.dropout, input_drop=args.input_drop) if args.avoid_features: base_model = None if args.dataset == "ogbn-mag": assert base_model is not None base_model = NARS(in_feats, num_hops, base_model, subset_list) if use_labels: label_model = GroupMLP(label_in_feats, args.num_hidden, n_classes, args.num_heads, args.label_mlp_layer, args.label_drop, residual=args.label_residual,) else: label_model = None model = SLEModel(base_model, label_model) return model
def __init__(self, mlp_kwargs, n1_kwargs, n2_kwargs, n3_kwargs=None): super(Medusa, self).__init__() self.name = 'medusa' self.n1 = CNN1D(**n1_kwargs) self.n2 = MobileNetV2(**n2_kwargs) self.n3 = None if n3_kwargs: self.n3 = CNN1D(**n3_kwargs) self.mlp = MLP(**mlp_kwargs)
def get_model(args): if args.model == 'mlp': model = MLP(num_classes=args.n_classes) elif args.model == 'resnet': model = ResNet(20, num_classes=args.n_classes) elif args.model == 'densenet': model = DenseNet(40, num_classes=args.n_classes) return model
def build_model(args, in_feats, n_hidden, n_classes, device, n_layers=1, backend='geometric', edge_index=None, num_nodes=None): if backend == 'geometric': if args.model == 'gs-mean': model = geo.GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == "gcn": model = geo.GCN(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == "gat": print("Warning, GAT doesn't respect n_layers") heads = [8, args.gat_out_heads] # Fixed head config n_hidden_per_head = int(n_hidden / heads[0]) model = geo.GAT(in_feats, n_hidden_per_head, n_classes, F.relu, args.dropout, 0.6, heads).to(device) elif args.model == "mlp": model = geo.MLP(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == "jknet": model = geo.JKNet(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == "sgnet": model = SGNet(in_channels=in_feats, out_channels=n_classes, K=n_layers).to(device) else: raise NotImplementedError else: if args.model == 'gs-mean': model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, 'mean').to(device) elif args.model == 'mlp': model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif args.model == 'mostfrequent': model = MostFrequentClass() elif args.model == 'gat': print("Warning, GAT doesn't respect n_layers") heads = [8, args.gat_out_heads] # Fixed head config # Div num_hidden by heads for same capacity n_hidden_per_head = int(n_hidden / heads[0]) assert n_hidden_per_head * heads[ 0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}" model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads, F.elu, 0.6, 0.6, 0.2, False).to(device) else: raise NotImplementedError("Model not implemented") return model
def GetModel(op, client_id, global_round): # initial global stop_round if op == 0: if model == 'cnn': if dataset == 'mnist' or dataset == 'fmnist': global_model = CNNMnist() elif dataset == 'cifar': global_model = CNNCifar() elif model == 'mlp': global_model = MLP(dim_in=img_size, dim_hidden=64, dim_out=10) elif model == 'densenet': global_model = torch.hub.load('pytorch/vision:v0.5.0', 'densenet121', pretrained=False) elif model == 'vgg19': global_model = torch.hub.load('pytorch/vision:v0.5.0', 'vgg19', pretrained=False) else: exit('Error: unrecognized model') encoing_string = base64.b64encode(pickle.dumps(global_model)).decode() clients_arr.remove(client_id) return encoing_string else: f_path_global = 'Models/global_model' + str(global_round) + '.pkl' if client_id == 1: while 1: filenums, files = GetFileNum(global_round) if filenums == client_nums: weights = [] for f in files: f_path = "Models/LocalModels/" + str( global_round) + '/' + f weights.append(torch.load(f_path)) new_global_model, cvg_flag = Average_weight(weights) if cvg_flag and stop_round is None: stop_round = global_round if global_round == total_global_round: print(global_loss_per_epoch, stop_round) print(f_path_global) save(new_global_model, f_path_global) #torch.save(new_global_model, f_path_global) with open(f_path_global, 'rb') as file: encoing_string = base64.b64encode(file.read()) clients_arr.remove(client_id) return encoing_string time.sleep(2) else: while 1: if os.path.isfile(f_path_global): with open(f_path_global, 'rb') as file: encoing_string = base64.b64encode(file.read()) clients_arr.remove(client_id) return encoing_string time.sleep(2)
def main(): save_dir = './results/' models = [ MLP( num_layers=NUM_LAYERS, in_dim=2, hidden_dim=HIDDEN_DIM, out_dim=1, activation='relu6', ), MLP( num_layers=NUM_LAYERS, in_dim=2, hidden_dim=HIDDEN_DIM, out_dim=1, activation='none', ), NAC( num_layers=NUM_LAYERS, in_dim=2, hidden_dim=HIDDEN_DIM, out_dim=1, ), NALU( num_layers=NUM_LAYERS, in_dim=2, hidden_dim=HIDDEN_DIM, out_dim=1 ), ] results = {} for fn_str, fn in ARITHMETIC_FUNCTIONS.items(): print('[*] Testing function: {}'.format(fn_str)) results[fn_str] = [] # dataset num_train=500, num_test=50, dim=100, num_sum=5, fn=fn, support=RANGE, )
def MLP_forecasting(dataset, inputDim, lr=1e-3, hiddenNum=50, outputDim=1, epoch=20, batchSize=30, plot_flag=False): # normalize time series # dataset = dataset.reshape(-1, 1) scaler = MinMaxScaler(feature_range=(0.0, 1.0)).fit(dataset) dataset = scaler.transform(dataset) # divide the series into training/testing samples # NOTE: Not RNN format train, test = util.divideTrainTest(dataset) trainX, trainY = util.createSamples(train, inputDim, RNN=False) testX, testY = util.createSamples(test, inputDim, RNN=False) print("trainX shape is", trainX.shape) print("trainY shape is", trainY.shape) print("testX shape is", testX.shape) print("testY shape is", testY.shape) # buil model and train MLP_model = MLP.MLP_Model(inputDim, hiddenNum, outputDim, lr) t1 = time.time() MLP_model.train(trainX, trainY, epoch, batchSize) t2 = time.time() - t1 print("train time is", t2) # forecasting trainPred = MLP_model.predict(trainX) testPred = MLP_model.predict(testX) # reverse the time series trainPred = scaler.inverse_transform(trainPred) trainY = scaler.inverse_transform(trainY) testPred = scaler.inverse_transform(testPred) testY = scaler.inverse_transform(testY) # evaluate MAE = eval.calcMAE(testY, testPred) print("test MAE", MAE) MRSE = eval.calcRMSE(testY, testPred) print("test RMSE", MRSE) SMAPE = eval.calcSMAPE(testY, testPred) print("test SMAPE", SMAPE) if plot_flag: util.plot(trainPred, trainY, testPred, testY) return trainPred, testPred, MAE, MRSE, SMAPE
def load_model(num_classes_to_predict): # vgg = SimpleNet(num_classes_to_predict) low_n_units = [32 * 32 * 3, 512, num_classes_to_predict] high_n_units = [32 * 32 * 3, 2048, 512, 256, num_classes_to_predict] lowest_n_units = [32 * 32 * 3, 128, num_classes_to_predict] low_n_units_mnist = [28 * 28, 256, num_classes_to_predict] high_n_units_mnist = [28 * 28, 512, 256, 128, 64, num_classes_to_predict] model = MLP(low_n_units) return model
def run(): # basics model = MLP([HIDDENS, HIDDENS, 10], config=config).to(DEVICE) # model = ResNet18(100, 20, config=config).to(DEVICE) tasks = get_rotated_mnist_tasks(NUM_TASKS, shuffle=True, batch_size=BATCH_SIZE) # optimizer = torch.optim.SGD(model.parameters(), lr=config['lr'], momentum=config['momentum']) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=config['gamma']) criterion = nn.CrossEntropyLoss().to(DEVICE) # hooks setup_experiment() # main loop time = 0 for current_task_id in range(1, NUM_TASKS+1): print("========== TASK {} / {} ============".format(current_task_id, NUM_TASKS)) train_loader = tasks[current_task_id]['train'] # task_lr = config['lr']*(config['gamma']**(current_task_id-1)) for epoch in range(1, EPOCHS+1): # train and save lr = max(config['lr']* config['gamma']**(current_task_id), config['lrlb']) print(lr) optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=config['momentum']) train_single_epoch(model, optimizer, train_loader, criterion, current_task_id) time += 1 # evaluate on all tasks up to now for prev_task_id in range(1, current_task_id+1): model = model.to(DEVICE) val_loader = tasks[prev_task_id]['test'] if epoch == EPOCHS: metrics = eval_single_epoch(model, val_loader, criterion, prev_task_id) log_metrics(metrics, time, prev_task_id) save_checkpoint(model, time) if prev_task_id == current_task_id: log_hessian(model, val_loader, time, prev_task_id) # save_checkpoint(model, time) # scheduler.step() end_experiment()
def test_cifar(data_path): print 'loading data' dl = CIFAR10DataLoader(cifar_file_path=data_path, load_dtype='float32') dp = LabeledMemoryDataProvider(data_loader=dl, batch_size=batch_size, max_gpu_train_samples=max_gpu_samples, max_gpu_valid_samples=max_gpu_samples, is_test=False, epochwise_shuffle=False, nvalid_samples=5000) opt = optimizers.SGD_Momentum_Optimizer() mlp = MLP(batch_size = batch_size, seed=seed, network_structure=mlp_ns, network_cost=mlp_cost) mlp.fit(data_provider=dp, optimizer=opt, train_epoch=training_epochs, early_stop=10, dump_freq=(50000//100+1)*10, train_mean=dp.get_data_stats(), batch_mean_subtraction=True) mlp.save('alex_cifar_26_model.joblib') # mlp.load('alex_cifar_26_model.joblib') # f = open(os.path.join(data_path,'test_batch'), 'rb') data = cPickle.load(f) f.close() test_set = data['data'] test_y = data['labels'] test_set = numpy.asarray(test_set, dtype='float32') tm = dp.get_data_stats() test_set -= tm pred = mlp.predict_from_memory_data(test_set) print 'Error: ', numpy.mean(pred!=test_y) dp = LabeledMemoryDataProvider(data_loader=dl, batch_size=batch_size, max_gpu_train_samples=max_gpu_samples, max_gpu_valid_samples=max_gpu_samples, is_test=True, epochwise_shuffle=False, nvalid_samples=0) pred = mlp.predict_from_data_provider(dp, train_mean=tm, batch_mean_subtraction=True) print 'Error: ', numpy.mean(pred!=test_y)
def test_finetune(): dl = MNISTDataLoader('/data/Research/datasets/mnist/mnist.pkl') dp = LabeledMemoryDataProvider(data_loader=dl, batch_size=batch_size, max_gpu_train_samples=max_gpu_samples, max_gpu_valid_samples=max_gpu_samples, is_test=False, epochwise_shuffle=False, nvalid_samples=10000) opt = optimizers.SGD_Rms_Optimizer() init_params = hickle.load('testmodel_ae.joblib') mlp = MLP(batch_size = batch_size, seed=seed, network_structure=mlp_ns, network_cost=mlp_cost, init_params=init_params) mlp.fit(data_provider=dp, optimizer=opt, train_epoch=finetune_epochs, early_stop=False, dump_freq=(50000//100+1)*10) mlp.save('mnist_mlp_model.joblib') f = open('/data/Research/datasets/mnist/mnist.pkl', 'rb') _, _, test_set = cPickle.load(f) f.close() pred = mlp.predict_from_memory_data(test_set[0]) print 'Error: ', numpy.mean(pred!=test_set[1]) dp = LabeledMemoryDataProvider(data_loader=dl, batch_size=batch_size, max_gpu_train_samples=max_gpu_samples, max_gpu_valid_samples=max_gpu_samples, is_test=True, epochwise_shuffle=False, nvalid_samples=0) pred = mlp.predict_from_data_provider(dp) print 'Error: ', numpy.mean(pred!=test_set[1])