def run_test(images): ''' Predict the characters cropped from the plate license. :param images: an 6 * 28 * 35 numpy array representing the 6 digitals an letters :return: ''' result = [] for i in images: tensor = torch.tensor([[i]], dtype=torch.float) # print(tensor.shape) # load pre-trained model cnn_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model', 'cnnnet_win.pkl') checkpoint = torch.load(cnn_dir, map_location='cpu') model = models.CNN() model.eval() model.class_to_idx = checkpoint['class_to_idx'] model.load_state_dict(checkpoint['state_dict']) # predict the characters output = model.forward(tensor) _, preds = torch.max(output, 1) print(preds.item()) for p in np.array(preds.cpu()): result.append(cat_to_class[model.class_to_idx[p]]) print(preds) return result
def initialize_model(model_name, hyperparams_path, device, pretrained=False, run_id=False, local=False): if model_name == 'cnn': image_size = 28 hyperparams = parse_hyperparams(hyperparams_path) hyperparams['num_classes'] = NUM_CLASSES hyperparams['image_size'] = image_size model = models.CNN(**hyperparams) requires_stroke_data = False elif model_name == 'cnn_rnn': hyperparams = parse_hyperparams(hyperparams_path) image_size = hyperparams['cnn_input_size'] hyperparams['rnn_input_size'] = STROKE_INPUT_SIZE hyperparams['num_classes'] = NUM_CLASSES model = models.CNN_RNN(**hyperparams) requires_stroke_data = True if pretrained: if not run_id: raise ValueError( "run_id is None. Need to specify run_id to resume training" ) model = load_model(model, model_name, run_id, local) requires_stroke_data = False logging.info('Hyperparams:') logging.info(json.dumps(hyperparams, indent=2)) logging.info(f"requires_stroke_data: {requires_stroke_data}") model = model.to(device) return model, image_size, requires_stroke_data
def build_and_train(self, n_epochs=20, **kwargs): config_defaults = { 'use_mse': False, 'use_bce': False, 'lr': 0.001, 'out_path': None, 'train_batch_size': 64, 'regularization': None, 'regularization_start_epoch': 2, 'l1': 0, 'l2': 0, 'bias_l1': 0, 'use_scrambling': False, 'use_overlay': False, 'use_elliptical': False, 'use_quadratic': False, 'quadratic_grad_scale': 1, 'mse_weighted': False, } model_kwargs = { k: v for k, v in kwargs.items() if k in models.CNN.config_defaults } cnn = models.CNN(**model_kwargs) print(cnn) # net architecture train_params = {**config_defaults, **kwargs} self.train_loop(cnn, n_epochs, train_params) return cnn
def main(config): """ 设备: cpu or GPU """ print("the current model is {}".format(config.model_name)) if config.model_name == "LR": device = torch.device("cpu") else: device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") n_gpu = torch.cuda.device_count() if torch.cuda.is_available(): print("device is cuda, # cuda is: ", n_gpu) else: print("device is cpu") """ 模型准备 """ train_loader, test_loader = datasets.minist_data(config) # 数据 if config.model_name == 'FNN': model = models.FNN(config).to(device) # 模型 elif config.model_name == "LR": model = models.LogisticRegressionMulti(config).to(device) elif config.model_name == "CNN": model = models.CNN().to(device) criterion = nn.CrossEntropyLoss() # 损失 optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate) # 优化算法 """ Train """ for epoch in range(1, config.epoch_num + 1): train(model, optimizer, criterion, train_loader, config.input_size, epoch, device) test(model, test_loader, config.input_size, criterion, device) torch.save(model.state_dict(), 'model.ckpt')
def CNN_train(test_fold, feat): """ Training CNN using extracted feature :param test_fold: test fold of 5-fold cross validation :param feat: which feature to use """ # 学习率衰减策略,每20个epoch衰减一次,变为0.1倍。 def scheduler(epoch): if epoch in [20, 40]: lr = K.get_value(model.optimizer.lr) K.set_value(model.optimizer.lr, lr * 0.1) print("lr changed to {}".format(lr * 0.1)) return K.get_value(model.optimizer.lr) # 读取特征数据 train_features, train_labels, test_features, test_labels = esc10_input.get_data( test_fold, feat) # 一些超参的配置 epoch = 60 num_class = 10 batch_size = 32 input_shape = (60, 65, 1) # 构建CNN模型 model = models.CNN(input_shape, num_class) # 回调函数 reduce_lr = LearningRateScheduler(scheduler) # 学习率衰减 logs = TensorBoard(log_dir='./log/fold{}/'.format(test_fold)) # 保存模型训练日志 checkpoint = ModelCheckpoint( './saved_model/cnn_{}_fold{}_best.h5'.format( feat, test_fold), # 保存在验证集上性能最好的模型 monitor='val_acc', verbose=1, save_best_only=True, mode='max', period=1) # 训练模型 model.fit(train_features, train_labels, batch_size=batch_size, nb_epoch=epoch, verbose=1, validation_split=0.1, callbacks=[checkpoint, reduce_lr, logs]) # 保存模型 model.save('./saved_model/cnn_{}_fold{}.h5'.format(feat, test_fold)) # 输出训练好的模型在测试集上的表现 score = model.evaluate(test_features, test_labels) print('Test score:', score[0]) print('Test accuracy:', score[1]) return score[1]
def build_model(config): model_type = config['model_type'] if model_type =='cnn': model = models.CNN(config) elif model_type =='bnn': model = models.BNN(config) elif model_type =='wage': model = models.WAGE(config) else: print("model_type={0} is not supported yet!".fortmat(model_type)) # Loss and Optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=config["lr"]) return model, criterion, optimizer
def __init__(self, **kwargs): super().__init__(**kwargs) with tf.variable_scope(self.scope): self.input = tf.placeholder(tf.float32, [None] + list(self.env.observation_space.shape), name='input') self.cnn_output = models.CNN(scope='cnn', convs=kwargs['convs'], hiddens=kwargs['hiddens'], inpt=self.input) self.mlp_output = models.MLP(scope='mlp', hiddens=kwargs['hiddens'], inpt=self.cnn_output)
def main(model_name="CNN", is_training=True): config = Config() if model_name == "CNN": model = models.CNN(config) elif model_name == "RNN": model = models.RNN(config) elif model_name == "RCNN": model = models.RCNN(config) else: model = models.FC(config) if is_training: model.train() else: model.restore_model() model.predict()
def build_model(self): """Creates and initializes the shared and controller models.""" if self.args.network_type == 'rnn': self.shared = models.RNN(self.args, self.dataset) elif self.args.network_type == 'cnn': self.shared = models.CNN(self.args, self.dataset) else: raise NotImplementedError(f'Network type ' f'`{self.args.network_type}` is not ' f'defined') self.controller = models.Controller(self.args) if self.args.num_gpu == 1: self.shared.cuda() self.controller.cuda() elif self.args.num_gpu > 1: raise NotImplementedError('`num_gpu > 1` is in progress')
def test(): # prepocssing the test data. test_transform = transforms.Compose([ transforms.Resize(28), transforms.Grayscale(num_output_channels=1), transforms.ToTensor() ]) # load the image fron the test set dictionary test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'images', 'characters', 'test') test_datasets = datasets.ImageFolder(test_dir, transform=test_transform) test_loader = torch.utils.data.DataLoader(test_datasets, batch_size=1) # load the pre trained model cnn_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'model', 'cnnnet_win.pkl') checkpoint = torch.load(cnn_dir, map_location='cpu') model = models.CNN() model.eval() # load the map relationship of class index to class name model.class_to_idx = checkpoint['class_to_idx'] # load model weights model.load_state_dict(checkpoint['state_dict']) results = [] for data in test_loader: # run the predition step on cpu images, labels = data images = images outputs = model(images) _, preds = torch.max(outputs, 1) print(preds) for p in np.array(preds.cpu()): results.append(cat_to_class[model.class_to_idx[p]]) print(results) return preds
def build_model(self): """Creates and initializes the shared and controller models.""" if self.args.network_type == 'rnn': self.shared = models.RNN(self.args, self.dataset) elif self.args.network_type == 'cnn': self.shared = models.CNN(self.args, self.dataset) else: raise NotImplementedError( 'Network type `{0}` is not defined'.format( self.args.network_type)) self.controller = models.Controller( self.args ) # 构建了一个orward:Embedding(130,100)->lstm(100,100)->decoder的列表,对应25个decoder if self.args.num_gpu == 1: self.shared.cuda() self.controller.cuda() elif self.args.num_gpu > 1: raise NotImplementedError('`num_gpu > 1` is in progress')
def main(args): ###### # 3.2 Processing of the data TEXT = data.Field(sequential=True, include_lengths=True, tokenize='spacy') LABEL = data.Field(sequential=False, use_vocab=False) train, val, test = data.TabularDataset.splits( path= "/Users/RobertAdragna/Documents/Third Year/Fall Term/MIE 324 - Introduction to Machine Intelligence/mie324/a4", train='train.tsv', validation='validation.tsv', test='test.tsv', format='tsv', skip_header=True, fields=[('text', TEXT), ('label', LABEL)]) # train_itr = data.BucketIterator(train, 64, sort_key=lambda x: len(x.TEXT), sort_within_batch=True, repeat=False) # val_itr = data.BucketIterator(val, 64, sort_key=lambda x: len(x.TEXT), sort_within_batch=True, repeat=False) # test_itr = data.BucketIterator(test, 64, sort_key=lambda x: len(x.TEXT), sort_within_batch=True, repeat=False) ###### train_iter, val_iter, test_iter = data.Iterator.splits( (train, val, test), sort_key=lambda x: len(x.text), sort_within_batch=True, repeat=False, batch_sizes=(64, 64, 64), device=-1) # train_iter, val_iter, test_iter = data.BucketIterator.splits( # (train, val, test), sort_key=lambda x: len(x.text), sort_within_batch=True, repeat=False, # batch_sizes=(64, 64, 64), device=-1) TEXT.build_vocab(train) vocab = TEXT.vocab vocab.load_vectors(torchtext.vocab.GloVe(name='6B', dim=100)) ###### # 5 Training and Evaluation base_model = models.Baseline(100, vocab) rnn_model = models.RNN(100, vocab, 100) cnn_model = models.CNN(100, vocab, 50, (2, 4)) train_func(rnn_model, train_iter, val_iter, test_iter, 20, "rnn")
def __init__(self, D_in=40, H=40, p=17, epochs=1000, batch_size=100, track_name='X_same_length_normalized', arch='fcnn', torch_seed=2): """ Parameters: ========================================================== D_in, H, p: int same as input to FCNN epochs: int number of epochs batch_size: int batch size track_name: str column name of track (the tracks should be of the same length) """ torch.manual_seed(torch_seed) self.D_in = D_in self.H = H self.p = p self.epochs = epochs self.batch_size = batch_size self.track_name = track_name self.torch_seed = torch_seed self.arch = arch torch.manual_seed(self.torch_seed) if self.arch == 'fcnn': self.model = models.FCNN(self.D_in, self.H, self.p) elif 'lstm' in self.arch: self.model = models.LSTMNet(self.D_in, self.H, self.p) elif 'cnn' in self.arch: self.model = models.CNN(self.D_in, self.H, self.p) elif 'attention' in self.arch: self.model = models.AttentionNet(self.D_in, self.H, self.p) elif 'video' in self.arch: self.model = models.VideoNet()
def build_model(self): """Creates and initializes the shared and controller models.""" if self.args.network_type == 'rnn': self.shared = models.RNN(self.args, self.dataset) elif self.args.network_type == 'cnn': print("----- begin to init cnn------") self.shared = models.CNN(self.args, self.dataset) # self.shared = self.shared.cuda() else: raise NotImplementedError(f'Network type ' f'`{self.args.network_type}` is not ' f'defined') print("---- begin to init controller-----") self.controller = models.Controller(self.args) #self.controller = self.controller.cuda() print("===begin to cuda") if True: print("cuda") self.shared.cuda() self.controller.cuda() print("finish cuda") elif self.args.num_gpu > 1: raise NotImplementedError('`num_gpu > 1` is in process')
def build_model(self): """Creates and initializes the shared and controller models.""" if self.args.network_type == 'rnn': self.shared = models.RNN(self.args, self.dataset) self.controller = models.Controller(self.args) elif self.args.network_type == 'micro_cnn': self.shared = models.CNN(self.args, self.dataset) self.controller = models.CNNMicroController(self.args) else: raise NotImplementedError(f'Network type ' f'`{self.args.network_type}` is not ' f'defined') if self.args.num_gpu == 1: if torch.__version__ == '0.3.1': self.shared.cuda() self.controller.cuda() else: self.shared.to(self.device) self.controller.to(self.device) elif self.args.num_gpu > 1: raise NotImplementedError('`num_gpu > 1` is in progress')
def others_predict(gold_test, ids_test, max_seq_length=256, batch_size=64, data_dir=DATA_DIR, models_dir=MODELS_BASE_DIR, results_dir=RESULTS_DIR, device=DEVICE): model_names = ["cnn", "han", "slstm", "clstm"] langs = ["en", "de"] embs = {"en": ["fasttext", "pubmed"], "de": ["fasttext"]} # load multi-label binarizer that contains classes and their labels mapping with open(os.path.join(data_dir, "mlb.pkl"), "rb") as rf: mlb = pkl.load(rf) for lang in langs: for emb in embs[lang]: if emb == "fasttext": embed_dim = 300 else: embed_dim = 400 hidden_dim = 300 for model_name in model_names: test_loader, dev_loader, V, Tv, C, T = get_test_data( model_name, lang, max_seq_len=256, batch_size=batch_size, data_dir=data_dir) vocab_size = V titles_vocab_size = Tv num_classes = C if model_name == "cnn": model = models.CNN(vocab_size, embed_dim, num_classes) elif model_name == "han": model = models.HAN(vocab_size, embed_dim, num_classes, h=hidden_dim, L=10, T=40, bidirectional=True) elif model_name == "slstm": model = models.SelfAttentionLSTM(vocab_size, embed_dim, num_classes, h=hidden_dim, bidirectional=True) else: if emb == "fasttext" and lang == "de": continue if lang == "de": hidden_dim = 150 T = T.to(device) model = models.ICDCodeAttentionLSTM(vocab_size, embed_dim, num_classes, T, Tv=titles_vocab_size, h=hidden_dim, bidirectional=True) model.to(device) model_name = "-".join([model_name, emb, lang]) print("______________________________________") print(" {} ".format(model_name)) print("______________________________________") model_dir = os.path.join(models_dir, model_name) model_file = os.path.join(model_dir, "model.pt") model.load_state_dict(torch.load(model_file)) model.eval() _, (_, test_preds, _, test_ids, _) = evaluate(test_loader, model, device, no_labels=True) testid2preds = { i: mlb.classes_[test_preds[idx].astype(bool)].tolist() for idx, i in enumerate(test_ids) } # official (include preds for doc ids where we do not even have gold labels) # this badly affects model as model make predictions for those examples as # well giving all as false positives, hurting precision badly. test_preds_official = { k: testid2preds[k] if k in testid2preds else [] for k in ids_test } preds_file = os.path.join(results_dir, model_name + "_preds_test.txt") generate_preds(preds_file, test_preds_official) out_file = os.path.join(results_dir, model_name + "_preds_test_eval.txt") results = challenge_eval("test", preds_file, out_file, data_dir) print("***** Test results (Original) *****") print(results) # here we only consider evaluating against examples where we have gold labels test_preds_fixed = { k: testid2preds[k] if k in testid2preds else [] for k in set(testid2preds.keys()).intersection( set(gold_test.keys())) } preds_file = os.path.join(results_dir, model_name + "_preds_test_fixed.txt") generate_preds(preds_file, test_preds_fixed) out_file = os.path.join( results_dir, model_name + "_preds_test_fixed_eval.txt") results = challenge_eval("test", preds_file, out_file, data_dir) print("***** Test results (Modified) *****") print(results)
})) # data preprocess cifar10_dataset_folder_path = args.data_path util.preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode) # load data valid_features, valid_labels = pickle.load( open('preprocess_validation.p', mode='rb')) # build the model tf.reset_default_graph() model = models.CNN(image_shape=(32, 32, 3), n_classes=10) # Inputs x = model.inputs y = model.label keep_prob = model.keep_prob # Model logits = model.conv_net(x, keep_prob) logits = tf.identity(logits, name='logits') # Loss and Optimizer cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) optimizer = tf.train.AdamOptimizer().minimize(cost)
def main(output_dim, train_bs, val_bs, test_bs, num_epochs, max_seq_length, learning_rate, warmup_proportion, early_stopping_criteria, num_layers, hidden_dim, bidirectional, dropout, filter_sizes, embedding_file, model_name, use_mongo, vm, subtask, _run): #Logger directory_checkpoints = f"results/checkpoints/{_run._id}/" directory = f"results/{_run._id}/" #Batch sizes batch_sizes = [int(train_bs), int(val_bs), int(test_bs)] batch_size = int(train_bs) if "BERT" in model_name: #Default = False, if BERT model is used then use_bert is set to True use_bert = True else: use_bert = False if vm == "google": directory = f"results-bert-google/{_run._id}/" elif vm == "aws": directory = f"results-bert-aws/{_run._id}/" #Data if use_bert: train_dataloader, val_dataloader, test_dataloader = get_data_bert( int(max_seq_length), batch_sizes, subtask) else: embedding_dim, vocab_size, embedding_matrix, train_dataloader, val_dataloader, test_dataloader = get_data( int(max_seq_length), embedding_file=embedding_file, batch_size=batch_size, subtask=subtask) #Model if model_name == "MLP": model = models.MLP(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, output_dim) if model_name == "MLP_Features": model = models.MLP_Features(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), 14, dropout, output_dim) print(model) elif model_name == "CNN": model = models.CNN(embedding_matrix, embedding_dim, vocab_size, dropout, filter_sizes, output_dim) print(model) elif model_name == "LSTM": model = models.LSTM(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, int(num_layers), bidirectional, output_dim) print(model) elif model_name == "LSTMAttention": model = models.LSTMAttention(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, int(num_layers), bidirectional, output_dim) print(model) elif model_name == "BERTFreeze": model = BertForSequenceClassification.from_pretrained( "bert-base-uncased", output_dim) for param in model.bert.parameters(): param.requires_grad = False print(param) print(param.requires_grad) print(model) elif model_name == "BERT": model = BertForSequenceClassification.from_pretrained( "bert-base-uncased", output_dim) print(model) elif model_name == "BERTLinear": model = models.BertLinear(hidden_dim, dropout, output_dim) print(model) elif model_name == "BERTLinearFreeze": model = models.BertLinearFreeze(hidden_dim, dropout, output_dim) print(model) elif model_name == "BERTLinearFreezeEmbeddings": model = models.BertLinearFreezeEmbeddings(hidden_dim, dropout, output_dim) print(model) elif model_name == "BERTLSTM": model = models.BertLSTM(hidden_dim, dropout, bidirectional, output_dim) print(model) elif model_name == "BERTNonLinear": model = models.BertNonLinear(dropout, output_dim) print(model) elif model_name == "BERTNorm": model = models.BertNorm(dropout, output_dim) print(model) model = model.to(device) #Loss and optimizer #optimizer = optim.Adam([{'params': model.parameters(), 'weight_decay': 0.1}], lr=learning_rate) optimizer = optim.Adam(model.parameters(), lr=learning_rate) loss_fn = F.cross_entropy #Scheduler #scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5, 50], gamma=0.1) #Training and evaluation print('Training and evaluation for {} epochs...'.format(num_epochs)) train_metrics, val_metrics = train_and_evaluate( num_epochs, model, optimizer, loss_fn, train_dataloader, val_dataloader, early_stopping_criteria, directory_checkpoints, use_bert, use_mongo) train_metrics.to_csv(directory + "train_metrics.csv"), val_metrics.to_csv( directory + "val_metrics.csv") #Test print('Testing...') load_checkpoint(directory_checkpoints + "best_model.pth.tar", model) #Add artifacts #ex.add_artifact(directory+"best_model.pth.tar") #ex.add_artifact(directory+"last_model.pth.tar") test_metrics = evaluate_model(model, optimizer, loss_fn, test_dataloader, device, use_bert) if use_mongo: log_scalars(test_metrics, "Test") test_metrics_df = pd.DataFrame(test_metrics) #test_metrics_df = pd.DataFrame(test_metrics, index=["NOT","OFF"]) print(test_metrics) test_metrics_df.to_csv(directory + "test_metrics.csv") id_nummer = f'{_run._id}' results = { 'id': id_nummer, 'loss': np.round(np.mean(val_metrics['loss']), 4), 'accuracy': test_metrics['accuracy'], 'recall': test_metrics['recall'], 'precision': test_metrics['precision'], 'f1': test_metrics['f1'], 'learning_rate': learning_rate, 'hidden_dim': hidden_dim, 'status': 'ok' } return results
video_output = video_model(video) image_output = img_model(image) dist = F.pairwise_distance(video_output, image_output) print(dist) pred = (dist < 10.0) print(label) correct += (pred == label).sum().float() total += len(label) acc = (100 * correct * 1.0 / total) return acc resnet = models.ResNet(block=models.BasicBlock, num_blocks=[3, 3, 3]) cnn3d = models.CNN3D() cnn = models.CNN() mlp = models.MLP() criterion = nn.MSELoss().cuda() optimizer = torch.optim.Adam(list(mlp.parameters()) + list(resnet.parameters()), lr=lr) # optimizer = torch.optim.SGD(list(cnn3d.parameters()) + list(resnet.parameters()), lr=lr, momentum=0.9, weight_decay=1e-4) task2_ds = matching_dataset(cat="whiteboard_spray", transforms=transform_train) val_len = len(task2_ds) // 10 train_len = len(task2_ds) - val_len train_ds, val_ds = torch.utils.data.random_split(task2_ds, [train_len, val_len]) train_loader = torch.utils.data.DataLoader(train_ds, 10, False, num_workers=10) val_loader = torch.utils.data.DataLoader(val_ds, 10, False, num_workers=10)
#corpus = data.read_corpus(constants.android_corpus_path) #embeddings, map_to_ids = data.embeddings(constants.android_embeddings_path) #id_to_tensors = data.map_corpus(corpus, map_to_ids) #train_corpus = data.read_corpus(constants.corpus_path) #train_id_to_tensors = data.map_corpus(train_corpus, map_to_ids) #train = data.get_train_data(constants.train_path, train_id_to_tensors) #dev = data.get_dev_data(constants.android_dev_path, id_to_tensors) #model = models.CNN(700, embeddings, 0.2) #models.train_model(train, dev, model, transfer=True) ##################################################### # Use this for running adversarial domain adaptation ##################################################### corpus = data.read_corpus(constants.android_corpus_path) embeddings, map_to_ids = data.embeddings(constants.android_embeddings_path) id_to_tensors = data.map_corpus(corpus, map_to_ids) train_corpus = data.read_corpus(constants.corpus_path) train_id_to_tensors = data.map_corpus(train_corpus, map_to_ids) train = data.get_android_data(constants.train_path, train_id_to_tensors, id_to_tensors) dev = data.get_dev_data(constants.android_dev_path, id_to_tensors) encoder = models.CNN(500, embeddings, 0.2) classifier = models.DomainClassifier(500, 300, 150) models.train_adversarial_model(train, dev, encoder, classifier)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits( (train_data, valid_data, test_data), batch_size=BATCH_SIZE, device=device) INPUT_DIM = len(TEXT.vocab) EMBEDDING_DIM = 100 N_FILTERS = 100 FILTER_SIZES = [3, 4, 5] OUTPUT_DIM = 1 DROPOUT = 0.5 if arch == "CNN": model = models.CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT) else: model = models.Binary_CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT) pretrained_embeddings = TEXT.vocab.vectors model.embedding.weight.data.copy_(pretrained_embeddings) optimizer = optim.Adam(model.parameters()) criterion = nn.BCEWithLogitsLoss() model = model.to(device) criterion = criterion.to(device) N_EPOCHS = 5
def train(args): import models import numpy as np np.random.seed(1234) if args.dataset == 'digits': n_dim, n_out, n_channels = 8, 10, 1 X_train, y_train, X_val, y_val = data.load_digits() elif args.dataset == 'mnist': n_dim, n_out, n_channels = 28, 10, 1 X_train, y_train, X_val, y_val, _, _ = data.load_mnist() elif args.dataset == 'svhn': n_dim, n_out, n_channels = 32, 10, 3 X_train, y_train, X_val, y_val = data.load_svhn() X_train, y_train, X_val, y_val = data.prepare_dataset(X_train, y_train, X_val, y_val) elif args.dataset == 'cifar10': n_dim, n_out, n_channels = 32, 10, 3 X_train, y_train, X_val, y_val = data.load_cifar10() X_train, y_train, X_val, y_val = data.prepare_dataset(X_train, y_train, X_val, y_val) elif args.dataset == 'random': n_dim, n_out, n_channels = 2, 2, 1 X_train, y_train = data.load_noise(n=1000, d=n_dim) X_val, y_val = X_train, y_train else: raise ValueError('Invalid dataset name: %s' % args.dataset) print 'dataset loaded, dim:', X_train.shape # set up optimization params p = { 'lr' : args.lr, 'b1': args.b1, 'b2': args.b2 } # create model if args.model == 'softmax': model = models.Softmax(n_dim=n_dim, n_out=n_out, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'mlp': model = models.MLP(n_dim=n_dim, n_out=n_out, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'cnn': model = models.CNN(n_dim=n_dim, n_out=n_out, n_chan=n_channels, model=args.dataset, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'kcnn': model = models.KCNN(n_dim=n_dim, n_out=n_out, n_chan=n_channels, model=args.dataset, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'resnet': model = models.Resnet(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'vae': model = models.VAE(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_batch=args.n_batch, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p, model='bernoulli' if args.dataset in ('digits', 'mnist') else 'gaussian') elif args.model == 'convvae': model = models.ConvVAE(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_batch=args.n_batch, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p, model='bernoulli' if args.dataset in ('digits', 'mnist') else 'gaussian') elif args.model == 'convadgm': model = models.ConvADGM(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_batch=args.n_batch, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p, model='bernoulli' if args.dataset in ('digits', 'mnist') else 'gaussian') elif args.model == 'sbn': model = models.SBN(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'adgm': model = models.ADGM(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_batch=args.n_batch, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p, model='bernoulli' if args.dataset in ('digits', 'mnist') else 'gaussian') elif args.model == 'hdgm': model = models.HDGM(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_batch=args.n_batch, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'dadgm': model = models.DADGM(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'dcgan': model = models.DCGAN(n_dim=n_dim, n_out=n_out, n_chan=n_channels, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) elif args.model == 'ssadgm': X_train_lbl, y_train_lbl, X_train_unl, y_train_unl \ = data.split_semisup(X_train, y_train, n_lbl=args.n_labeled) model = models.SSADGM(X_labeled=X_train_lbl, y_labeled=y_train_lbl, n_out=n_out, n_superbatch=args.n_superbatch, opt_alg=args.alg, opt_params=p) X_train, y_train = X_train_unl, y_train_unl else: raise ValueError('Invalid model') # train model model.fit(X_train, y_train, X_val, y_val, n_epoch=args.epochs, n_batch=args.n_batch, logname=args.logname)
def run_cnn_exp(data, embedding_matrix, token_to_idx, seed=0, weight_decay=0.0, lr=0.0001, max_len=51, batch_size=50, idx_to_label=['negative', 'neutral', 'positive'], embedding_freeze=True, embedding_normalize=True, obj='loss', measures=['loss', 'macro_f1', 'acc', 'avgrecall'], epoches=100, silent=False, cuda=-1): # set seed for reproduciable seed = seed torch.backends.cudnn.deterministic = True torch.cuda.manual_seed_all(seed) random.seed(seed) torch.manual_seed(seed) np.random.seed(seed) # Load data into numpy format train_np_sentences, train_np_labels = data_helper.map_to_num( data['train'][0], data['train'][1], token_to_idx, idx_to_label=idx_to_label, max_len=max_len) dev_np_sentences, dev_np_labels = data_helper.map_to_num( data['dev'][0], data['dev'][1], token_to_idx, idx_to_label=idx_to_label, max_len=max_len) test_np_sentences, test_np_labels = data_helper.map_to_num( data['test'][0], data['test'][1], token_to_idx, idx_to_label=idx_to_label, max_len=max_len) # create sampler to solve imbalance of training data train_num_count = [0] * len(idx_to_label) for label in train_np_labels: train_num_count[label] += 1 sample_weights = [0.0] * len(train_np_labels) for i, label in enumerate(train_np_labels): sample_weights[i] = len(train_np_labels) / train_num_count[label] sampler = torch.utils.data.sampler.WeightedRandomSampler( sample_weights, len(sample_weights)) # create iter for train, dev and test train_iter = DataLoader(data_helper.BasicDataset(train_np_sentences, train_np_labels), batch_size=batch_size, sampler=sampler) dev_iter = DataLoader(data_helper.BasicDataset(dev_np_sentences, dev_np_labels), batch_size=batch_size) test_iter = DataLoader(data_helper.BasicDataset(test_np_sentences, test_np_labels), batch_size=batch_size) cnn = models.CNN(embedding_matrix, max_len, embedding_freeze=embedding_freeze, embedding_normalize=embedding_normalize, num_classes=len(idx_to_label)) if cuda != -1: cnn.cuda(cuda) # start training criterion = torch.nn.CrossEntropyLoss(size_average=False) optimizer = torch.optim.Adam(cnn.custom_params, lr=lr, weight_decay=weight_decay) obj_value = 0.0 final_metrics = { 'loss': 0.0, 'macro_f1': 0.0, 'acc': 0.0, 'avgrecall': 0.0 } for epoch in range(epoches): start_time = time.time() cnn.train() train_sum_loss = 0.0 train_count = 0 train_predict = [] train_gold = [] for batch in train_iter: optimizer.zero_grad() sentences = batch['sentence'] labels = batch['label'] if cuda != -1: sentences = sentences.cuda() labels = labels.cuda() sentences = Variable(sentences) labels = Variable(labels) outputs = cnn(sentences) _, outputs_label = torch.max(outputs, 1) for label in outputs_label.data: train_predict.append(int(label)) for label in labels.data: train_gold.append(int(label)) loss = criterion(outputs, labels) loss.backward() optimizer.step() train_sum_loss += loss.data[0] train_count += labels.shape[0] train_metrics_result = metrics.evaluation_metrics( train_gold, train_predict, measures=measures, idx_to_label=idx_to_label) train_metrics_result['loss'] = train_sum_loss / train_count if not silent: output_str = "[{}/{}]\ntrain\t".format(epoch + 1, epoches) for key in measures: output_str += "{}={:.4f}\t".format(key, train_metrics_result[key]) print(output_str) cnn.eval() dev_sum_loss = 0.0 dev_count = 0 dev_predict = [] dev_gold = [] for batch in dev_iter: optimizer.zero_grad() sentences = batch['sentence'] labels = batch['label'] if cuda != -1: sentences = sentences.cuda() labels = labels.cuda() sentences = Variable(sentences) labels = Variable(labels) outputs = cnn(sentences) _, outputs_label = torch.max(outputs, 1) for label in outputs_label.data: dev_predict.append(int(label)) for label in labels.data: dev_gold.append(int(label)) loss = criterion(outputs, labels) dev_sum_loss += loss.data[0] dev_count += labels.shape[0] dev_metrics_result = metrics.evaluation_metrics( dev_gold, dev_predict, measures=measures, idx_to_label=idx_to_label) dev_metrics_result['loss'] = dev_sum_loss / dev_count if not silent: output_str = "dev\t".format(epoch + 1, epoches) for key in measures: output_str += "{}={:.4f}\t".format(key, dev_metrics_result[key]) print(output_str) test_sum_loss = 0.0 test_count = 0 test_predict = [] test_gold = [] for batch in test_iter: optimizer.zero_grad() sentences = batch['sentence'] labels = batch['label'] if cuda != -1: sentences = sentences.cuda() labels = labels.cuda() sentences = Variable(sentences) labels = Variable(labels) outputs = cnn(sentences) _, outputs_label = torch.max(outputs, 1) for label in outputs_label.data: test_predict.append(int(label)) for label in labels.data: test_gold.append(int(label)) loss = criterion(outputs, labels) test_sum_loss += loss.data[0] test_count += labels.shape[0] test_metrics_result = metrics.evaluation_metrics( test_gold, test_predict, measures=measures, idx_to_label=idx_to_label) test_metrics_result['loss'] = test_sum_loss / test_count if not silent: output_str = "test\t".format(epoch + 1, epoches) for key in measures: output_str += "{}={:.4f}\t".format( key, round(test_metrics_result[key], 5)) print(output_str) # output time if not silent: print("cost time:{}".format(time.time() - start_time)) # early stop procedure if epoch == 0: obj_value = dev_metrics_result[obj] final_metrics = test_metrics_result else: if obj != 'loss': if dev_metrics_result[obj] > obj_value: obj_value = dev_metrics_result[obj] final_metrics = test_metrics_result else: if dev_metrics_result[obj] < obj_value: obj_value = dev_metrics_result[obj] final_metrics = test_metrics_result return obj_value, final_metrics, cnn.embed.weight.data.cpu().numpy()
criterion = torch.nn.NLLLoss().to(device) def weights_init(model): # Official init from torch tutorial for m in model.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight) elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) elif isinstance(m, nn.Linear): nn.init.constant_(m.bias, 0) encoder = models.CNN(opt.imgH, input_channels, opt.nh) decoder = models.AttentionDecoder(opt.nh, nclass, dropout_p=0.1, max_length=opt.max_width) encoder.apply(weights_init) decoder.apply(weights_init) encoder = encoder.to(device) decoder = decoder.to(device) # continue training or use the pretrained model to initial the parameters of the encoder and decoder if opt.encoder: print('loading pretrained encoder model from %s' % opt.encoder) encoder.load_state_dict(torch.load(opt.encoder)) if opt.decoder:
train(model, device, args, optimazer, criterior, train_dataset, test_dataset) input, target = next(iter(train_dataset)) print(model(input.to(device))) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--batch_size', type=int, default=12, help='Size of batch during training') parser.add_argument('--num_workers', type=int, default=2, help='Count workers for data loading') parser.add_argument('--path_to_DataFile', type=str, default='ecg_data_200.json', help='Path to data file') parser.add_argument('--lr', type=float, default=0.001, help='Learning rate') parser.add_argument('--eph', type=int, default=1, help='Count of epoches') args = parser.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = models.CNN().to(device) main(args, model)
test_x, test_y = data_process.to_tensor(test_x, test_y) #%% ########## Load Data ########## # change to dataset train_dataset = Data.TensorDataset(train_x, train_y) test_dataset = Data.TensorDataset(test_x, test_y) # load data loader_train = Data.DataLoader( dataset=train_dataset, # torch TensorDataset format batch_size=Args.batch_size, # mini batch size shuffle=True, # shuffle or not num_workers=0, # multi-worker for load data ) loader_test = Data.DataLoader(test_dataset, Args.batch_size) del ECG_data, ECG_label, train_x, test_x, train_y, test_y #%% ########## Create model ########## cnn = models.CNN().cuda() if Args.cuda else models.CNN() # optimizer optimizer = torch.optim.Adam(cnn.parameters(), lr=Args.learn_rate) # loss function loss_func = nn.CrossEntropyLoss() # evaluate Accuracy = [] F1 = [] # save Best_result = [0, [], []] #%% ########## Training ########## if Args.show_plot: fig = plt.figure(1) plt.ion() print('Start Training') for epoch in range(Args.epoch):
def main(): parser = argparse.ArgumentParser(description='Chainer-Tutorial: CNN') parser.add_argument('--batch_size', '-b', type=int, default=128, help='Number of samples in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=100, help='Number of times to train on data set') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID: -1 indicates CPU') parser.add_argument( '--file-path', '-fp', type=str, default= '/media/tasuku/Samsung_T3/workspace/imdb_data/imdb_crop/imdb_crop/pkl_folder', help='Path to pickle file') parser.add_argument('--file-name', '-fn', type=str, default='imdb_data_50.pkl', help='Data set (image, label)') parser.add_argument('--white', '-w', type=bool, default=False, help='Preprocess whitening') args = parser.parse_args() train, test, age_stats = utils.get_data(args.file_path, args.file_name, simple=True, white=args.white, split=0.3) print("Age distribution of data is : {}".format(age_stats)) train_iter = chainer.iterators.SerialIterator(train, args.batch_size) test_iter = chainer.iterators.SerialIterator(test, args.batch_size, repeat=False, shuffle=False) model = L.Classifier(models.CNN(4)) if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() model.to_gpu() optimizer = chainer.optimizers.RMSprop(lr=0.001, alpha=0.9) optimizer.setup(model) updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch')) trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) trainer.extend(extensions.dump_graph('main/loss')) report_params = [ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ] trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport(report_params)) trainer.extend(extensions.ProgressBar()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Run trainer trainer.run()
def main(output_dim, train_bs, val_bs, test_bs, num_epochs, max_seq_length, learning_rate, warmup_proportion, early_stopping_criteria, num_layers, hidden_dim, bidirectional, dropout, filter_sizes, embedding_file, model_name, use_mongo, _run): #Logger directory = f"results/{_run._id}/" #Batch sizes batch_sizes = [int(train_bs), int(val_bs), int(test_bs)] batch_size = int(train_bs) if "BERT" in model_name: #Default = False, if BERT model is used then use_bert is set to True use_bert = True else: use_bert = False #Data if use_bert: train_dataloader, val_dataloader, test_dataloader = get_data_bert( int(max_seq_length), batch_sizes) else: embedding_dim, vocab_size, embedding_matrix, train_dataloader, val_dataloader, test_dataloader = get_data_features( int(max_seq_length), embedding_file=embedding_file, batch_size=batch_size) #Model if model_name == "MLP": model = models.MLP(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, output_dim) if model_name == "MLP_Features": model = models.MLP_Features(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), 13, dropout, output_dim) print(model) elif model_name == "CNN": model = models.CNN(embedding_matrix, embedding_dim, vocab_size, dropout, filter_sizes, output_dim) print(model) elif model_name == "LSTM": model = models.LSTM(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, int(num_layers), bidirectional, output_dim) print(model) elif model_name == "LSTMAttention": model = models.LSTMAttention(embedding_matrix, embedding_dim, vocab_size, int(hidden_dim), dropout, int(num_layers), bidirectional, output_dim) print(model) elif model_name == "BERT": model = BertForSequenceClassification.from_pretrained( "bert-base-uncased", output_dim) print(model) elif model_name == "BERTLinear": model = models.BertLinear(hidden_dim, dropout, output_dim) print(model) elif model_name == "BERTLSTM": model = models.BertLSTM(hidden_dim, dropout, output_dim) print(model) model = model.to(device) #Loss and optimizer optimizer = optim.Adam(model.parameters(), lr=learning_rate) loss_fn = F.cross_entropy #Training and evaluation print('Training and evaluation for {} epochs...'.format(num_epochs)) train_metrics, val_metrics = train_and_evaluate( num_epochs, model, optimizer, loss_fn, train_dataloader, val_dataloader, early_stopping_criteria, directory, use_bert, use_mongo) train_metrics.to_csv(directory + "train_metrics.csv"), val_metrics.to_csv( directory + "val_metrics.csv") #Test print('Testing...') load_checkpoint(directory + "best_model.pth.tar", model) test_metrics = evaluate_model(model, optimizer, loss_fn, test_dataloader, device, use_bert) if use_mongo: log_scalars(test_metrics, "Test") test_metrics_df = pd.DataFrame(test_metrics) print(test_metrics) test_metrics_df.to_csv(directory + "test_metrics.csv") id_nummer = f'{_run._id}' results = { 'id': id_nummer, 'loss': np.round(np.mean(val_metrics['loss']), 4), 'accuracy': test_metrics['accuracy'], 'recall': test_metrics['recall'], 'precision': test_metrics['precision'], 'f1': test_metrics['f1'], 'learning_rate': learning_rate, 'hidden_dim': hidden_dim, 'status': 'ok' } return results
import random import datetime import sklearn.model_selection as sk from sklearn.preprocessing import StandardScaler from operator import mul import copy import pandas as pd import models import fns import cv2 cap = cv2.VideoCapture(0) hyperpars = {'drop_rate': 0.0, 'dense_size': 64, 'conv_filters': [16, 32]} cnn = models.CNN((128, 128, 3), 3, hyperpars, name="test") # test = models.CNN_multi((128,128,3), 3, hyperpars, name = "test") # test.build_branch(128,[16,32]) # exit(0) cnn.build_layers() # cnn.training = False # original_files = yield_files() colour = ['red', 'purple', 'green'] count = ['single', 'double', 'triple'] shape = ['pill', 'diamond', 'squiggle'] fill = ['empty', 'grid', 'solid'] model_name = 'colour' v = colour
import torch import actions import models import data_manipulation as dm import auxiliary as aux from auxiliary import printc import texts import configurations as config import os os.chdir('modules') net = models.CNN() def options(): learning_rate, minibatch, trainset, testset, epochs = config.get_configs() printc("cyan","1. Visualizar arquitetura.") printc("cyan","2. Treinar rede.") printc("cyan","3. Testar rede.") printc("cyan","4. Predição individual.") printc("cyan","5. Carregar modelo.") printc("cyan","6. Salvar modelo.") printc("cyan","7. Resetar pesos.") printc("cyan","8. Configurações.") printc("cyan","9. Créditos.") printc("cyan","10. Sair.\n") option = input("Insira a opção desejada: ") aux.clear_screen() if(option=='1'): actions.print_model(net, minibatch) aux.press_enter()