def train(self, max_epochs, batch_size, val_split, callbacks): """Train model, using specified epoch number, batch_size and callbacks. Parameters ---------- max_epochs : The maximum number of epochs to train for. batch_size : Batch size used during training, i.e. the number of sentences. callbacks : List of Keras callbacks. """ if self.use_aux_input: _X_train, _y_train, _X_aux = shuffle_data(self.X_train, self.y_train, self.X_aux) else: _X_train, _y_train = shuffle_data(self.X_train, self.y_train) if self.visualize: for epoch in range(max_epochs): if self.use_aux_input: self.model.fit([_X_train, _X_aux], _y_train, batch_size=batch_size, epochs=epoch + 1, initial_epoch=epoch, validation_split=val_split, callbacks=callbacks) attention_output = self.attention_layer_model.predict( [self.sample_sequence.reshape(1, self.num_timesteps), self.sample_aux.reshape(1, 3)], batch_size=1) else: self.model.fit(_X_train, _y_train, batch_size=batch_size, epochs=epoch + 1, initial_epoch=epoch, validation_split=val_split, callbacks=callbacks) attention_output = self.get_attention_output self.attention_history = np.append( self.attention_history, [attention_output[0, -self.sample_length:, 0]], axis=0) else: if self.use_aux_input: self.model.fit([_X_train, _X_aux], _y_train, batch_size=batch_size, epochs=max_epochs, validation_split=0.15, callbacks=callbacks) else: self.model.fit(_X_train, _y_train, batch_size=batch_size, epochs=max_epochs, validation_split=0.1, callbacks=callbacks) print('Training done\n')
def shuffle_data_by_partition(self, partition): """ Shuffle the training or validation data """ if partition == 'train': self.train_audio_paths, self.train_durations, self.train_texts = shuffle_data( self.train_audio_paths, self.train_durations, self.train_texts) elif partition == 'valid': self.valid_audio_paths, self.valid_durations, self.valid_texts = shuffle_data( self.valid_audio_paths, self.valid_durations, self.valid_texts) else: raise Exception("Invalid partition. " "Must be train/validation")
def train_sfm(model, x_train, y_train, epochs=10): optimizer = optim.Adam(model.parameters()) criterion = nn.MSELoss() X, Y = shuffle_data(x_train, y_train) x_len = X.shape[1] X = ToVariable(X) Y = ToVariable(Y) for epoch in range(0, epochs): h, c, re_s, im_s, time = model.init_state() for step in range(0, x_len): x = X[:, step, :] y = Y[:, step, :] x = x.unsqueeze(1) output, h, c, re_s, im_s, time = model(x, h, c, re_s, im_s, time) h = h.data c = c.data re_s = re_s.data im_s = im_s.data time = time.data loss = criterion(output.squeeze(0), y) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() print('Epoch: ', epoch + 1, '| step: ', step + 1, '| Loss: ', loss.detach()) torch.save(model, 'model/model2.pkl')
def train(self, train_data, dev_data): self.sess = tf.Session() self.sess.run(self.init) for epoch in range(self.num_epochs): print("------- epoch: %i --------" % epoch) train_data = shuffle_data(train_data) total_batch = int(len(train_data["vec_0"]) / self.batch_size) for i in range(total_batch): vec_0, vec_1, label, len_0, len_1 = self.get_minibatch( train_data, self.batch_size * i, self.batch_size * (i + 1)) dpool_index = dynamic_pooling_index(len_0, len_1, self.seq_len) feed_dict = { self.model.sent0: vec_0, self.model.sent1: vec_1, self.model.label: label, self.model.dpool_index: dpool_index, } _, loss = self.sess.run([self.train_op, self.model.loss], feed_dict) if i % self.display_step_freq == 0: auc, acc, loss = self.evaluate(dev_data) print("step: %i\t auc: %f\t acc: %f\t loss: %f" % (i, auc, acc, loss)) self.saver.save(self.sess, self.output_path + self.model_name + ".ckpt")
def fit(self, X, y, n_iterations=4000): self._initialize_parameters(X) #tune parameters for n iterations for i in self.bar(range(n_iterations)): if self.gradient_descent: #move against the gradient of the loss function with #respect to the parameters to minimize the loss if self.opt_type == "GD": y_pred = self.sigmoid(X.dot(self.param)) self.param -= self.learning_rate * -(y - y_pred).dot(X) elif self.opt_type == "SGD": X_shuffle, y_shuffle = shuffle_data(X, y) for i in range(self.n_sample): y_pred = self.sigmoid(X_shuffle[i, :].dot(self.param)) self.param -= self.learning_rate * -( y_shuffle[i] - y_pred) * X_shuffle[i, :] else: y_pred = self.sigmoid(X.dot(self.param)) #make a diagonal matrix of the sigmoid gradient column vector diag_gradient = make_diagonal( self.sigmoid.gradient(X.dot(self.param))) #batch opt self.param = np.linalg.pinv(X.T.dot(diag_gradient).dot(X)).dot( X.T).dot( diag_gradient.dot(X).dot(self.param) + y - y_pred)
def train(epoch): model.train() data = shuffle_data(train_data, 1) total_loss = 0.0 for num, i in enumerate(range(0, len(data), args.batch_size)): one = data[i:i + args.batch_size] query, _ = padding([x[0] for x in one], max_len=50) passage, _ = padding([x[1] for x in one], max_len=350) answer = pad_answer([x[2] for x in one]) query, passage, answer = torch.LongTensor(query), torch.LongTensor( passage), torch.LongTensor(answer) if args.cuda: query = query.cuda() passage = passage.cuda() answer = answer.cuda() optimizer.zero_grad() loss = model([query, passage, answer, True]) loss.backward() total_loss += loss.item() optimizer.step() if (num + 1) % args.log_interval == 0: print( '|------epoch {:d} train error is {:f} eclipse {:.2f}%------|' .format(epoch, total_loss / args.log_interval, i * 100.0 / len(data))) total_loss = 0
def load_data(self, b_unfold_label): file_path = self.file_path coco = COCO(file_path) # display # categories and supercategories cats = coco.loadCats(coco.getCatIds()) nms = [cat['name'] for cat in cats] #print('COCO categories: \n{}\n'.format(' '.join(nms))) self.num_classes = len(cats) images = [] labels = [] for cat in cats: #catIds = coco.getCatIds(catNms=cat['name']) catIds = cat['id'] # print(catIds) imgIds = coco.getImgIds(catIds=catIds) img = coco.loadImgs(imgIds) # labels.extend([(catIds[0] % 10000 - 1) for i in range(len(img))]) labels.extend([(catIds % 10000 - 1) for i in range(len(img))]) images.extend(img) if len(images) == 0: images = coco.dataset['images'] if b_unfold_label: labels = unfold_label(labels=labels, classes=len(np.unique(labels))) #assert len(images) == len(labels) self.images = np.array(images) self.labels = np.array(labels) self.file_num_train = len(self.labels) print('data num loaded:', self.file_num_train) if self.stage is 'train': self.images, self.labels = shuffle_data(samples=self.images, labels=self.labels)
def make_dataset(category_list): superdir_path = '/mnt/dc/web_food_imgs_f/' global NB_CLASS NB_CLASS = len(category_list) category_label_mapping = {} train_food_img_list = [] train_food_img_labels = [] val_food_img_list = [] val_food_img_labels = [] # 1. read all image name list for idx, category in enumerate(category_list): category_label_mapping[category] = idx dir_path = os.path.join(superdir_path, category) img_list = [ os.path.join(dir_path, f) for f in os.listdir(dir_path) if f.endswith(('.jpg', 'jpeg', '.png', '.bmp', '.JPG', 'JPEG', '.PNG', '.BMP')) ] # split to train & validation train_list, val_list = utils.split_by_KFold(img_list, nb_splits=10) train_food_img_list = train_food_img_list + train_list val_food_img_list = val_food_img_list + val_list train_food_img_labels = train_food_img_labels + [idx] * len(train_list) val_food_img_labels = val_food_img_labels + [idx] * len(val_list) print(category_label_mapping) # 2. shuffle return utils.shuffle_data( train_food_img_list, train_food_img_labels), val_food_img_list, val_food_img_labels
def __init__(self, inputs, labels, output_size=100, learning_rate=0.001, test_size=0.33, dropout=0.0, hidden_layers=0, hidden_sizes=[], epochs=10, batch_size=100): # intialize InceptionNetV3 self.IncepV3 = InceptionV3(weights='imagenet', include_top=False) # split data into training and testing datasets self.train_inputs, self.train_labels, self.test_inputs, self.test_labels = utils.shuffle_data( inputs, labels, test_size) # derived parameters self.input_length = len(self.train_inputs[0]) # optional parameters that have defaulted values self.learning_rate = learning_rate self.output_size = output_size self.dropout = dropout self.hidden_layers = hidden_layers self.hidden_sizes = hidden_sizes self.epochs = epochs self.batch_size = batch_size # computation graph construction self.construct_graph()
def do_eval(test_data_path, shuffle=False): if FLAGS.load_model is None: raise ValueError("You need to specify the model location by --load_model=[location]") # Load Testing Data question_1, question_2, labels = get_input_from_csv(test_data_path) if shuffle: question_1, question_2, labels = shuffle_data(question_1, question_2, labels) # Load Pre-trained Model if FLAGS.best_glove: import en_core_web_md nlp = en_core_web_md.load() # load best-matching version for Glove else: nlp = spacy.load('en') embedding_matrix = load_glove_embeddings(nlp.vocab, n_unknown=FLAGS.num_unknown) # shape=(1071074, 300) tf.logging.info('Build model ...') esim = ESIM(embedding_matrix, FLAGS.max_length, FLAGS.num_hidden, FLAGS.num_classes, FLAGS.keep_prob, FLAGS.learning_rate) if FLAGS.load_model: model = esim.build_model(FLAGS.load_model) else: raise ValueError("You need to specify the model location by --load_model=[location]") # Convert the "raw data" to word-ids format && convert "labels" to one-hot vectors q1_test, q2_test = convert_questions_to_word_ids(question_1, question_2, nlp, max_length=FLAGS.max_length, tree_truncate=FLAGS.tree_truncate) labels = to_categorical(np.asarray(labels, dtype='int32')) scores = model.evaluate([q1_test, q2_test], labels, batch_size=FLAGS.batch_size, verbose=1) print("=================== RESULTS =====================") print("[*] LOSS OF TEST DATA: %.4f" % scores[0]) print("[*] ACCURACY OF TEST DATA: %.4f" % scores[1])
def train(self, train_data, dev_data): self.sess = tf.Session() self.sess.run(self.init) for epoch in range(self.num_epochs): print("------- epoch: %i --------" % epoch) train_data = shuffle_data(train_data) total_batch = int( math.ceil(len(train_data["vec_0"]) / self.batch_size)) for i in range(total_batch): vec_0, vec_1, label = self.get_minibatch( train_data, self.batch_size * i, self.batch_size * (i + 1)) feed_dict = { self.model.sent_a: vec_0, self.model.sent_b: vec_1, self.model.label: label, } _ = self.sess.run(self.train_op, feed_dict) if i % self.display_freq == 0: auc, acc, loss = self.evaluate(dev_data) print("step: %i\t auc: %f\t acc: %f\t loss: %f" % (i, auc, acc, loss)) print("saving in %s" % self.model_path + self.modelname + ".ckpt") self.saver.save(self.sess, self.model_path + self.model_name + ".ckpt")
def train_lstm(model, x_train, y_train, epochs=10): optimizer = optim.RMSprop(model.parameters()) criterion = nn.MSELoss() X, Y = shuffle_data(x_train, y_train) x_len = X.shape[1] X = ToVariable(X) Y = ToVariable(Y) for epoch in range(0, epochs): h, c = model.init_state() for step in range(0, x_len): x = X[:, step, :].view(-1, 1, 1) y = Y[:, step, :] out_put, h, c = model(x, h, c) h = h.data c = c.data optimizer.zero_grad() loss = criterion(out_put, y) loss.backward(retain_graph=True) optimizer.step() print('Epoch: ', epoch + 1, '| step: ', step + 1, '| Loss: ', loss.detach()) torch.save(model, 'model/model.pkl')
def explore_weak_clf(samples, labels, num_pos, num_neg, weights, weights_ada, train_param, sam_ori_index): """ :param samples: lÃst of samples :param labels: :param weights: :return selected_clf, min_err: the classifier is added to strong classifier and its err value """ # selected_feature = random.sample(range(0, len(samples[0]) - 1), NUM_FEATURES) logger.info('Start exploring weak classifier') selected_feature = random.sample(range(0, len(samples[0])), NUM_FEATURES) min_err = sys.maxsize selected_clf = None feature_idx = None num_samples = len(samples) samples, labels, weights, weights_ada, sam_ori_index = utils.shuffle_data( samples, labels, weights, weights_ada, sam_ori_index) count = 1 for idx in selected_feature: samples_feature = list() for i in range(len(samples)): samples_feature.append(samples[i][idx]) # l = [feature for feature in samples[idx][selected_feature[idx]] for idx in range(len(samples))] samples_feature = np.array(samples_feature) samples_feature = samples_feature.reshape((num_samples, 72)) labels = labels.reshape((num_samples, )) weights = weights.reshape((num_samples, )) weights_ada = weights_ada.reshape((num_samples, )) # split data # data = train_test_split(samples_feature, labels, weights, test_size=0.3) # samples_feature_train = data[0] # samples_feature_test = data[1] # labels_train = data[2] # labels_test = data[3] # weights_train = data[4] # weights_test = data[5] # svm_clf = svm_generator(samples_feature, labels) # clf_err = 1 - svm_clf.score(samples_feature, labels, weights) svm_clf = svm_generator(samples_feature, labels, num_pos, num_neg, weights_ada, train_param) # clf_err = 1 - svm_clf.score(samples_feature_test, labels_test, weights_test) clf_err = 0 # logger.info(f'sum weight {weights_ada.sum()}') preds = svm_clf.predict(samples_feature) clf_err += get_err_weak_clf(samples_feature, labels, preds, weights_ada) # logger.info(f'{count}.err: {clf_err}, index: {idx}') logger.info("%s.err: %s, index: %s" % (count, clf_err, idx)) if clf_err < min_err: min_err = clf_err selected_clf = svm_clf feature_idx = idx count += 1 return selected_clf, min_err, feature_idx
def load_dataset_from_disk(args): datas = DatasetDict() if args.train_file is not None and args.train: train_json = json.load(open(args.train_file)) train_json, dev_json = shuffle_data(train_json, args.split) datas["train"] = Dataset.from_dict(convert_data_structure(train_json)) datas["dev"] = Dataset.from_dict(convert_data_structure(dev_json)) logger.info(f"Load train data number: {datas['train'].num_rows}") logger.info(f"Load dev data number: {datas['dev'].num_rows}") if args.eval and not args.train: train_json = json.load(open(args.train_file)) train_json, dev_json = shuffle_data(train_json, args.split) datas['dev'] = Dataset.from_dict(convert_data_structure(dev_json)) logger.info(f"Load dev data number: {datas['dev'].num_rows}") if args.predict_file is not None and args.predict: predict_json = json.load(open(args.predict_file)) datas["predict"] = Dataset.from_dict(convert_data_structure(predict_json, ispredict=True)) logger.info(f"Load predict data number: {datas['predict'].num_rows}") return datas
def eval(self, input, input_chr, labels): """ Evalutates the model using Accuracy, Precision, Recall and F1 metrics. :param input: Input of shape [batch_size, timestep, vector_dim] :param :return: """ if input.size == 0: return None logging.info("Evaluating on the validation set...") num_batches = input_chr.shape[0] // self.batch_size input, input_chr, labels = utils.shuffle_data( [input, input_chr, labels]) acc, prec, rec, f1, loss_sum = 0, 0, 0, 0, 0 for b in range(num_batches): word_1 = input[b * self.batch_size:(b + 1) * self.batch_size][:, 0] char_1 = input_chr[b * self.batch_size:(b + 1) * self.batch_size][:, 0] word_2 = input[b * self.batch_size:(b + 1) * self.batch_size][:, 1] char_2 = input_chr[b * self.batch_size:(b + 1) * self.batch_size][:, 1] label = labels[b * self.batch_size:(b + 1) * self.batch_size] loss, pred = self.sess.run( [self.loss, self.softmax], { self.word_embedding_input_1: word_1, self.chr_embedding_input_1: char_1, self.word_embedding_input_2: word_2, self.chr_embedding_input_2: char_2, self.labels: label }) # Update metric a, p, r, f = utils.calc_metric(np.argmax(pred, axis=1), np.argmax(label, axis=1)) acc += a prec += p rec += r f1 += f loss_sum += loss logging.info("Accuracy {:.3f}%".format(acc / num_batches * 100)) logging.info("Weighted Macro Precision {:.3f}%".format( prec / num_batches * 100)) logging.info("Weighted Macro Recall {:.3f}%".format(rec / num_batches * 100)) logging.info(" Weighted Macro F1 {:.3f}%".format(f1 / num_batches * 100)) logging.info("Average loss {:.5f}\n".format(loss_sum / num_batches)) return loss_sum / num_batches
def yield_dataset(X, y=None, batch_size=32, shuffle=True): if y is not None: assert (len(X) == len(y)) if shuffle: X, y = shuffle_data(X, y) # Only complete batches are submitted for i in range(len(X) // batch_size): if y is not None: yield X[i * batch_size:(i + 1) * batch_size], y[i * batch_size:(i + 1) * batch_size] else: yield X[i * batch_size:(i + 1) * batch_size]
def process_data(): images, labels, label_dict = flower_photos_data.load_flower_datasets() assert np.max(images[0]) <= 1, 'The image should be scaled to 0-1' images, labels = utils.shuffle_data(images, labels) labels_onehot = utils.one_hot_encode(labels) train_images, train_labels, valid_images, valid_labels, test_images, test_labels = \ utils.split_data(images, labels_onehot, train_size=0.8, valid_size=0.1, test_size=0.1) os.mkdir('data/flower_npy') np.save('data/flower_npy/train_images.npy', train_images) np.save('data/flower_npy/train_labels.npy', train_labels) np.save('data/flower_npy/valid_images.npy', valid_images) np.save('data/flower_npy/valid_labels.npy', valid_labels) np.save('data/flower_npy/test_images.npy', test_images) np.save('data/flower_npy/test_labels.npy', test_labels)
def get_data(self, dataset, b_unfold_label): self.ecgs = dataset[0] self.labels = dataset[1] # shift the labels to start from 0 : already done. if b_unfold_label: self.labels = unfold_label(labels=self.labels, classes=len(np.unique(self.labels))) self.file_num_train = len(self.labels) print('data num loaded:', self.file_num_train) if self.stage is 'train': self.ecgs, self.labels = shuffle_data(samples=self.ecgs, labels=self.labels)
def split_data(corpus, split_idx=0.1): """ Split the tweets from the corpus into trainig and test data. :param corpus: TweetCorpusReader :param split_idx: double :return: """ # Load the corpus data and labels for classification X, y = corpus.tweets, corpus.categories # Shuffle the tweets X, y = shuffle_data(X, y) split_idx = int(len(X) * split_idx) X_test, y_test = X[:split_idx], y[:split_idx] X_train, y_train = X[split_idx:], y[split_idx:] return X_train, y_train, X_test, y_test
def eval(model, char, word, label, batch_size): """ Evaluates the model using standard evaluation metrics :param model: Trained Model :param char: List of char embeddings :param word: List of word embeddings :param label: List of labels :param batch_size: Batch size :return: """ logging.info("Evaluating on the validation set...") char, word, label = utils.shuffle_data(char, word, label) num_batches = char.shape[0] // batch_size acc, prec, rec, f1 = 0, 0, 0, 0 for b in range(num_batches): chr_b = char[b * batch_size:(b + 1) * batch_size] word_b = word[b * batch_size:(b + 1) * batch_size] label_b = label[b * batch_size:(b + 1) * batch_size] loss, unary_scores, trans_params = model.sess.run( [model.loss, model.logits, model.trans_params], {model.char_embedding_input: chr_b, model.word_embedding_input: word_b, model.labels: label_b}) # Get CRF output scores true_batch = [] for tf_unary_scores_ in unary_scores: # Compute the highest scoring sequence. viterbi_sequence, _ = tf.contrib.crf.viterbi_decode( tf_unary_scores_, trans_params) true_batch.append(viterbi_sequence) # Update metric a, p, r, f = calc_metric(np.array(true_batch), np.argmax(label_b, axis=2)) acc += a prec += p rec += r f1 += f logging.info("Accuracy {:.3f}%".format(acc / num_batches * 100)) logging.info("Macro Precision {:.3f}%".format(prec / num_batches * 100)) logging.info("Macro Recall {:.3f}%".format(rec / num_batches * 100)) logging.info("Macro F1 {:.3f}%\n".format(f1 / num_batches * 100))
def create_seperate_dataset(data, num_classes): """ Creates dataset in a form: sentence_repr -> one_hot_humor_label Shuffles the data before returning. :param data: Train or dev data :return: Shuffled word, char and labels """ word_repr, char_repr, labels = [], [], [] for doc in data: for sent in doc: word_repr.append(sent[0]) char_repr.append(sent[1]) labels.append(utils.int_to_one_hot(int(sent[2]), num_classes)) word_repr, char_repr, labels = np.array(word_repr), \ np.array(char_repr, dtype=np.int32), \ np.array(labels) return utils.shuffle_data([word_repr, char_repr, labels])
def prepare_wjs_data(process_data, test_size): """ Prepares the WJS Treebank dataset for training. Pipeline consists of embeddings creation and one-hot label encoding. :param process_data: :param test_size: :return: """ # Setup paths path_prefix = os.path.join( constants.WJS_DATA, "wjs_treebank_timest" + str(constants.TIMESTEP) + "_") word_embeddings_path = path_prefix + "word_embeddings.pkl" char_id_mappings_path = path_prefix + "char_id_mappings.pkl" char_embeddings_path = path_prefix + "char_embeddings.pkl" labels_path = path_prefix + "onehot_labels.pkl" if process_data: wjs_data = parse_WJS(constants.WJS_DATA_DIR) input_process.embed_words(wjs_data, export_file=word_embeddings_path) input_process.create_char_mappings( wjs_data, export_file=char_embeddings_path, mappings_export_file=char_id_mappings_path) input_process.encode_labels(wjs_data, export_file=labels_path) # Load data from disk treebank = utils.load_pickle(word_embeddings_path) pos_tags = utils.load_pickle(labels_path) chr_embds = utils.load_pickle(char_embeddings_path) chr_id_mappings = utils.load_pickle(char_id_mappings_path) # Shuffle chr_embds, treebank, pos_tags = utils.shuffle_data(chr_embds, treebank, pos_tags) # Split train_chr, valid_chr, test_chr, train_word, valid_word, test_word, train_label, \ valid_label, test_label = utils.split_data(chr_embds, treebank, pos_tags, test_size=test_size) return train_chr, valid_chr, test_chr, train_word, valid_word, \ test_word, train_label, valid_label, test_label, chr_id_mappings
def train(self, x, y, batch_size=32, n_epochs=100, shuffle=True, verbose=True, print_interval=1): losses = [] n_batches = x.shape[0] // batch_size for epoch in range(n_epochs): x, y = shuffle_data(x, y) x_batches = np.split(x, n_batches) y_batches = np.split(y, n_batches) epoch_losses = [] for x_batch, y_batch in zip(x_batches, y_batches): dict = {self.input: x_batch, self.y: y_batch} self._sess.run(self._train_step, feed_dict=dict) loss = self._sess.run(self._loss, feed_dict=dict) epoch_losses.append(loss) if verbose and (epoch % print_interval) == 0: print "Epoch %d/%d, loss = %f."%(epoch, n_epochs, float(np.mean(epoch_losses))) losses.append(np.mean(epoch_losses)) return losses
def get_images_labels_batch(self,batch_size=None): if batch_size is not None: self.batch_size = batch_size images = [] labels = [] for index in range(self.batch_size): self.current_index += 1 # void over flow if self.current_index > self.file_num_train - 1: self.current_index %= self.file_num_train self.images, self.labels = shuffle_data(samples=self.images, labels=self.labels) #img = cv2.imread(self.images[self.current_index]['file_name']) img = Image.open(self.images[self.current_index]['file_name']) img = img.convert('RGB') img = transform_train(img) img = np.array(img) images.append(img) labels.append(self.labels[self.current_index]) return np.array(images), np.array(labels)
def get_images_labels_batch(self): images = [] labels = [] for index in range(self.batch_size): self.current_index += 1 # void over flow if self.current_index > self.file_num_train - 1: self.current_index %= self.file_num_train self.images, self.labels = shuffle_data(samples=self.images, labels=self.labels) img = Image.open('data/PACS/pacs_data/' + self.images[self.current_index]) img = img.convert('RGB') img = transform_train(img) img = np.array(img) images.append(img) labels.append(self.labels[self.current_index]) return np.array(images), np.array(labels)
def get_ecgs_labels_batch(self): ecgs = [] labels = [] for index in range(self.batch_size): self.current_index += 1 # void over flow if self.current_index > self.file_num_train - 1: self.current_index %= self.file_num_train self.ecgs, self.labels = shuffle_data(samples=self.ecgs, labels=self.labels) ecgs.append(self.ecgs[self.current_index]) labels.append(self.labels[self.current_index]) ecgs = np.stack(ecgs) labels = np.stack(labels) return ecgs, labels
def load_data(self, b_unfold_label): file_path = self.file_path f = h5py.File(file_path, "r") self.images = np.array(f['images']) self.labels = np.array(f['labels']) f.close() # shift the labels to start from 0 self.labels -= np.min(self.labels) if b_unfold_label: self.labels = unfold_label(labels=self.labels, classes=len(np.unique(self.labels))) assert len(self.images) == len(self.labels) self.file_num_train = len(self.labels) print('data num loaded:', self.file_num_train) if self.stage is 'train': self.images, self.labels = shuffle_data(samples=self.images, labels=self.labels)
def load_data(self, b_unfold_label): file_path = self.file_path images = [] labels = [] with open(file_path, 'r') as file_to_read: while True: lines = file_to_read.readline() if not lines: break pass image, label = [i for i in lines.split()] images.append(image) labels.append(int(label) - 1) pass if b_unfold_label: labels = unfold_label(labels=labels, classes=len(np.unique(labels))) self.images = np.array(images) self.labels = np.array(labels) self.file_num_train = len(self.labels) if self.stage is 'train': self.images, self.labels = shuffle_data(samples=self.images, labels=self.labels)
model_path = os.path.join(checkpoint_path, model_name) if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) print(checkpoint_path) # Load dataset data = loadmat(data_path) x_train = data['x_train'] y_train = np.squeeze(data['y_train']) x_test = data['x_test'] y_test = np.squeeze(data['y_test']) data_size = y_train.shape[0] # shuffle_index = utils.shuffle_data(data_size, random_seed=random_seed) shuffle_index = utils.shuffle_data(data_size) x_train = x_train[shuffle_index] y_train = y_train[shuffle_index] print(x_train.shape) nb_classes = len(np.unique(y_train)) samples = x_train.shape[3] channels = x_train.shape[2] # Build Model if model_used == 'EEGNet': model = old_models.EEGNet(nb_classes=nb_classes, Chans=channels, Samples=samples) elif model_used == 'DeepConvNet':
valid_losses = [np.inf] # training process with tf.Session() as sess: # contruct a tensorflow saver node saver = tf.train.Saver() # initialize model.init(sess) iter = 0 summary_writer = tf.summary.FileWriter(logdir) # start training for i in range(total_epoch): print('----------epoch {}-----------'.format(i)) training_data = shuffle_data(training_data) iiter = 0 for j in range(i % 892, num_train, batch_size ): # take the data in a batch: length is batch_size print('------data_iter {}-------'.format(iiter)) iter += 1 # here is training feed_dict = get_batch_feed_dict(model, j, batch_size, training_data, global_inputs, global_attn_states) _, merged_summary = sess.run( [model.phs['train_op'], model.phs['summary']], feed_dict) # summary_writer.add_summary(merged_summary, iter) if iter % save_log_iter == 0: summary_writer.add_summary(merged_summary, iter)
def main(args): batch_size = args.batch_size lr = args.lr num_epochs = args.num_epochs max_len = args.max_len word_hidden_size_ = args.word_hidden_size sent_hidden_size_ = args.sent_hidden_size embed_size = 1024 print("load data") startt = time.clock() recipe_context, recipe_images, recipe_question, recipe_choice, recipe_answer = load_cleaned_data( 'train_cleaned.json') recipe_context_valid, recipe_images_valid, recipe_question_valid, recipe_choice_valid, recipe_answer_valid = load_cleaned_data( 'val_cleaned.json') recipe_context_test , recipe_images_test, recipe_question_test , recipe_choice_test , recipe_answer_test = load_cleaned_data( 'test_cleaned.json') recipe_context = recipe_context[:7824] recipe_images = recipe_images[:7824] recipe_question = recipe_question[:7824] recipe_choice = recipe_choice[:7824] recipe_answer = recipe_answer[:7824] recipe_context_valid = recipe_context_valid[:960] recipe_images_valid = recipe_images_valid[:960] recipe_question_valid = recipe_question_valid[:960] recipe_choice_valid= recipe_choice_valid[:960] recipe_answer_valid = recipe_answer_valid[:960] end = time.clock() print(str(end-startt)) model = QAnet(max_len,model_dim = 5122) if args.load_model: model.load_state_dict(torch.load(args.load_model)) model.eval() print("LOAD Model:", args.load_model) optimizer = optim.Adam(model.parameters(),lr = lr, weight_decay=0.0001) criterion = nn.CrossEntropyLoss() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) criterion = criterion.to(device) max_val_acc = 0.0 for epoch in tqdm(range(num_epochs)): print(epoch) print("shuffle data") # startt = time.clock() train_context_new, train_question_new, train_choice_new, train_answer_new = shuffle_data(recipe_context, recipe_question, recipe_choice, recipe_answer) train_context, train_question, train_choice, train_answer = split_batch(batch_size, train_context_new, train_question_new, train_choice_new, train_answer_new) val_context_new, val_question_new, val_choice_new, val_answer_new = shuffle_data(recipe_context_valid, recipe_question_valid, recipe_choice_valid, recipe_answer_valid) val_context, val_question, val_choice, val_answer = split_batch(batch_size, val_context_new, val_question_new, val_choice_new, val_answer_new) test_context_new, test_question_new, test_choice_new, test_answer_new = shuffle_data(recipe_context_test, recipe_question_test, recipe_choice_test, recipe_answer_test) test_context, test_question, test_choice, test_answer = split_batch(batch_size,test_context_new, test_question_new, test_choice_new, test_answer_new) # end = time.clock() # print(str(end - startt)) train_loss, train_acc = train(model,train_context,train_question,train_answer,train_choice,optimizer,criterion) val_loss ,val_acc = evaluate(model,val_context,val_question,val_answer, val_choice, criterion) test_loss, test_acc = evaluate(model, test_context, test_question, test_answer, test_choice, criterion) # log_data(args.log_path,train_loss,train_acc,val_loss,val_acc) print(f'| Epoch: {epoch + 1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc * 100:.2f}% | Val. Loss: {val_loss:.3f} | Test. Acc: {test_acc * 100:.2f}%| Test. Loss:{test_loss:.3f} | test. Acc: {test_loss * 100:.2f}%')