def eval_models(models_paths: list, path_to_data: str): if len(models_paths) == 0: return 0.0 # getting test loader ds = FashionMnistHandler(path_to_data, False) ds.download() ds.load() # noise parameters are not relevant since test loader shouldn't have noise _, _, test_loader = ds.get_noisy_loaders(0, '1', 0.2, 128, 128, 128) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') test_acc = [] for model_file in models_paths: # creating model checkpoint = torch.load(model_file, map_location=device) model_name = model_file.split("/")[-1] # loading from checkpoint model = CNNModel() model.load_state_dict(checkpoint['model_state_dict']) model.to(device) loss_fn = torch.nn.CrossEntropyLoss() # evaluating _, acc = Solver.eval(model, device, loss_fn=loss_fn, data_loader=test_loader) test_acc.append(acc) print(f"Model {model_name} has {acc:.4f} acc in test dataset") return test_acc
def main(config): # Create test dataloader test_ds_cfg = config['test_data_loader']['args'] test_dataset = MNISTDataset(test_ds_cfg['root'], test_ds_cfg['csv_file'], False, transforms=transforms) test_loader = DataLoader(test_dataset, test_ds_cfg['batch_size'], test_ds_cfg['shuffle'], num_workers=test_ds_cfg['num_workers']) # Create classification model model = CNNModel() # Create criterion (loss function) criterion = getattr(loss_md, config['loss']) # Create metrics for evaluation metrics = [getattr(loss_md, x) for x in config['metrics']] # Create optimizer optimizer = getattr(otpm, config['optimizer']['name'])( model.parameters(), **config['optimizer']['args']) # Create learning rate scheduler lr_scheduler = getattr(otpm.lr_scheduler, config['lr_scheduler']['name'])\ (optimizer, **config['lr_scheduler']['args']) # Create train procedure: classification trainer csf_trainer = ClassificationTrainer(config, model, criterion, metrics, optimizer, lr_scheduler) csf_trainer.test(test_loader)
def main(): args = sys.argv batch_size = 128 epochs = 100 maxlen = 300 model_path = 'models/cnn_model.h5' num_words = 40000 num_label = 2 x, y = load_dataset('data/amazon_reviews_multilingual_JP_v1_00.tsv') x = preprocess_dataset(x) x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42) vocab = build_vocabulary(x_train, num_words) x_train = vocab.texts_to_sequences(x_train) x_test = vocab.texts_to_sequences(x_test) x_train = pad_sequences(x_train, maxlen=maxlen, truncating='post') x_test = pad_sequences(x_test, maxlen=maxlen, truncating='post') emb_flg = args[0] if emb_flg == 't': wv = load_fasttext('../chap08/models/cc.ja.300.vec.gz') wv = filter_embeddings(wv, vocab.word_index, num_words) else: wv = None model = CNNModel(num_words, num_label, embeddings=wv).build() model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc']) callbacks = [ EarlyStopping(patience=3), ModelCheckpoint(model_path, save_best_only=True) ] model.fit(x=x_train, y=y_train, batch_size=batch_size, epochs=epochs, validation_split=0.2, callbacks=callbacks, shuffle=True) model = load_model(model_path) api = InferenceAPI(model, vocab, preprocess_dataset) y_pred = api.predict_from_sequences(x_test) print('precision: {:.4f}'.format( precision_score(y_test, y_pred, average='binary'))) print('recall : {:.4f}'.format( recall_score(y_test, y_pred, average='binary'))) print('f1 : {:.4f}'.format(f1_score(y_test, y_pred, average='binary')))
def train(): pos_data_path = '../dataset/weibo60000/pos60000_utf8.txt_updated' pos_x, pos_y = read_pos_data(pos_data_path) print(len(pos_x)) print(len(pos_y)) # print(pos_y) neg_data_path = '../dataset/weibo60000/neg60000_utf8.txt_updated' neg_x, neg_y = read_neg_data(neg_data_path) print(len(neg_x)) print(len(neg_y)) # print(neg_y) train_pos_x = pos_x[:41025] train_pos_y = pos_y[:41025] val_pos_x = pos_x[41025:52746] val_pos_y = pos_y[41025:52746] test_pos_x = pos_x[52746:] test_pos_y = pos_y[52746:] train_neg_x = neg_x[:41165] train_neg_y = neg_y[:41165] val_neg_x = neg_x[41165:52926] val_neg_y = neg_y[41165:52926] test_neg_x = neg_x[52926:] test_neg_y = neg_y[52926:] train_x, train_y = concate_data(train_pos_x, train_pos_y, train_neg_x, train_neg_y) val_x, val_y = concate_data(val_pos_x, val_pos_y, val_neg_x, val_neg_y) test_x, test_y = concate_data(test_pos_x, test_pos_y, test_neg_x, test_neg_y) print('The number of train-set:', len(train_x)) # print(len(train_y)) print('The number of val-set:', len(val_x)) # print(len(val_y)) print('The number of test-set:', len(test_x)) # print(len(test_y)) embedding = BERTEmbedding('../dataset/chinese_L-12_H-768_A-12', sequence_length=100) print('embedding_size', embedding.embedding_size) # print(embedding.model.output model = CNNModel(embedding) model.fit(train_x, train_y, val_x, val_y, batch_size=128, epochs=20, fit_kwargs={'callbacks': [tf_board_callback]}) model.evaluate(test_x, test_y) model.save('./model/cnn_bert_model')
def __init__(self, input_path, output_path): self.input_path = input_path self.output_path = output_path set_seed() if not os.listdir(input_path): raise NameError("Input directory is empty") if not os.path.isdir(output_path): os.mkdir(output_path) # For speed optimization, all algorithms are defined here self.backSub = cv.createBackgroundSubtractorMOG2() self.CNN = CNNModel()
def __init__(self, dm): self._action_spec = array_spec.BoundedArraySpec(shape=(), dtype=np.int32, minimum=0, maximum=1, name='action') self._observation_spec = array_spec.BoundedArraySpec( # !TODO check range of values shape=(1610, ), dtype=np.float32, minimum=0.0, maximum=1.0, name='observation') self._state = np.zeros( (1610, ), dtype=np.float32 ) # observation, should be updated and passes to timestep self._episode_ended = False self._n_queries = 0 self.total_reward = 0 self._counter = 0 self.dm = dm self.cnn_model = CNNModel(self.dm.get_x_shape(), NUM_CLASSES, epochs=5) self.aux_model = AuxModel(self.cnn_model._model, 4) # Flatten layer
import numpy from anna import util from anna.datasets import supervised_dataset from models import CNNModel print('Start') pid = os.getpid() print('PID: {}'.format(pid)) f = open('pid', 'wb') f.write(str(pid) + '\n') f.close() model = CNNModel('experiment', './', learning_rate=1e-2) monitor = util.Monitor(model) # Loading CIFAR-10 dataset print('Loading Data') data_path = '/data/cifar10/' reduced_data_path = os.path.join(data_path, 'reduced', 'cifar10_100') train_data = numpy.load(os.path.join(reduced_data_path, 'train_X_split_0.npy')) train_labels = numpy.load( os.path.join(reduced_data_path, 'train_y_split_0.npy')) test_data = numpy.load('/data/cifar10/test_X.npy') test_labels = numpy.load('/data/cifar10/test_y.npy') train_dataset = supervised_dataset.SupervisedDataset(train_data, train_labels) test_dataset = supervised_dataset.SupervisedDataset(test_data, test_labels)
batch_size = 64 num_epochs = 100 #################################################################################### ####################### Dataloading ################################################ train_dataset = Fashion_MNIST_Data("./data/train.csv") trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4) test_dataset = Fashion_MNIST_Data("./data/test.csv") testloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False, num_workers=4) classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot') #################################################################################### ####################### Network #################################################### net = CNNModel() net.to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(),lr=learning_rate) #################################################################################### ####################### Training ################################################### for epoch in range(num_epochs): running_loss = 0.0 for i, (images, labels) in enumerate(trainloader): images, labels = images.to(device), labels.to(device) optimizer.zero_grad() outputs = net(images) loss = criterion(outputs, torch.max(labels,1)[1]) loss.backward()
def loadModel(target, model_fl): #model_fl = getBestModelFileName(model_fl) #print(model_fl) params = model_fl.split("_") model_name = params[0] # print("lo", params) model = None #outnode, model_name, target, opt, learn_r, epch, dropout_keep_rate, save_model = False if model_name == "ImageNetInceptionV2": model = ImageNetInceptionV2(2, params[0], params[1], params[2], float(params[3]), int(params[4]), float(params[5]), False) else: model = CNNModel(2, params[0], params[1], params[2], float(params[3]), int(params[4]), int(params[5]), float(params[6]), False) model_fl = model_fl.split(".data")[0] print("{}/{}".format(model_files_path, model_fl)) model.load("{}/{}".format(model_files_path, model_fl)) chembl_target_threshold_dict = getModelThresholds( "deepscreen_models_hyperparameters_performance_results.tsv") compound_smiles_dict = getSMILEsFromFileWithHeader(test_fl) comp_id_list = list(compound_smiles_dict.keys()) num_of_comps = len(comp_id_list) #print(num_of_comps) pred_count = 0 count = 0 print("ACTIVE PREDICTIONS:{}".format(target)) for comp_id in comp_id_list: count += 1 #print(count) test_data = [] try: img_arr = drawPictureandReturnImgMatrix( TEMP_IMG_OUTPUT_PATH, compound_smiles_dict[comp_id], comp_id) test_data.append( [np.array(img_arr / 255.0), np.array([0, 0]), comp_id]) except: pass test_x = np.array([i[0] for i in test_data ]).reshape(-1, IMG_SIZE, IMG_SIZE, 1) test_comp_name = [i[2] for i in test_data] test_predictions = model.predict(test_x) test_predictions = test_predictions[:, 0] test_pred_labels = [ int(round(i, 2) >= chembl_target_threshold_dict[target]) for i in test_predictions ] for i in range(len(test_predictions)): if test_pred_labels[i] == 1: pred_count += 1 #print("ACTPRED\t{}\t{}\t{}".format(model_fl, target, test_comp_name[i])) print("{}".format(test_comp_name[i]))
if __name__ == "__main__": parser = argparse.ArgumentParser( prog='cifar10_checkpoint_checker', description='Script to select best performing checkpoint on CIFAR-10.') parser.add_argument("checkpoint_dir", help='Folder containing all .pkl checkpoint files.') args = parser.parse_args() checkpoint_dir = args.checkpoint_dir if not os.path.exists(checkpoint_dir): raise Exception('Checkpoint directory does not exist.') checkpoint_list = sorted(os.listdir(checkpoint_dir)) model = CNNModel('xxx', './') model.fc4.dropout = 0.0 model._compile() num_channels = model.conv1.filter_shape[0] filter_size = model.conv1.filter_shape[1] # Get iterators for cifar10 test set test_iterator = load_cifar10_data() # Create object to local contrast normalize a batch. # Note: Every batch must be normalized before use. normer = util.Normer2(filter_size=filter_size, num_channels=num_channels) test_accuracies = [] for i, checkpoint_file in enumerate(checkpoint_list):
def trainModelTarget(model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, rotate, save_model): #target_only_alpnum = ''.join(ch for ch in target if ch.isalnum()) model = None # ImageNetInceptionv2 # AlexNetModel # CNNModel2 # CNNModel if model_name=="ImageNetInceptionV2": model = ImageNetInceptionV2(2, model_name, target, optimizer, learning_rate, epch, dropout_keep_rate, save_model) elif model_name == "AlexNetModel": model = AlexNetModel(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, save_model=False) elif model_name == "CNNModel": model = CNNModel(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, dropout_keep_rate, save_model) elif model_name == "CNNModel2": model = CNNModel2(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, dropout_keep_rate, save_model) else: pass #train, test = getTrainDataBinary("{}/{}".format(Y_IMG_PATH,target), target ) train, validation, test = constructDataMatricesForMUVDataset(TEMP_IMG_OUTPUT_PATH, target, rotate) train_comp_name = [i[2] for i in train] X = [] for i in train: if i[0].shape!=(): X.append(i[0]) X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1) Y = [] for i in train: if i[0].shape != (): Y.append(i[1]) validation_x = [] for i in validation: if i[0].shape!=(): validation_x.append(i[0]) validation_x = np.array(validation_x).reshape(-1, IMG_SIZE, IMG_SIZE, 1) validation_y = [] for i in validation: if i[0].shape != (): validation_y.append(i[1]) validation_comp_name = [] for i in validation: if i[0].shape != (): validation_comp_name.append(i[2]) test_x = [] for i in test: if i[0].shape!=(): test_x.append(i[0]) test_x = np.array(test_x).reshape(-1, IMG_SIZE, IMG_SIZE, 1) test_y = [] for i in test: if i[0].shape != (): test_y.append(i[1]) test_comp_name = [] for i in test: if i[0].shape != (): test_comp_name.append(i[2]) test_x = np.array([i[0] for i in test]).reshape(-1,IMG_SIZE,IMG_SIZE,1) test_y = [i[1] for i in test] test_comp_name = [i[2] for i in test] if save_model: model.fit(X, Y, n_epoch=epch, validation_set=({'input': validation_x}, {'targets': validation_y}), show_metric=True, batch_size=32, snapshot_step=200, snapshot_epoch=True, run_id="{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_id".format(model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, rotate, save_model)) else: model.fit(X, Y, n_epoch=epch, validation_set=({'input': validation_x}, {'targets': validation_y}), show_metric=True, batch_size=32) test_predictions = model.predict(test_x) validation_predictions = model.predict(validation_x) test_predictions = test_predictions[:,0] validation_predictions = validation_predictions[:,0] #print(validation_predictions) test_y = [i[0] for i in test_y] validation_y = [i[0] for i in validation_y] threshold = 1.00 # f1score, mcc, accuracy, precision, recall, tp, fp, tn, fn, threshold best_test_accuracy_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] best_test_f1score_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] best_test_mcc_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] best_validation_accuracy_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] best_validation_f1score_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] best_validation_mcc_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1] validation_auc = roc_auc_score(validation_y, validation_predictions) validation_auprc = average_precision_score(validation_y, validation_predictions) # validation_bedroc = bedroc_score(validation_y, validation_predictions) test_auc = roc_auc_score(test_y, test_predictions) test_auprc = average_precision_score(test_y, test_predictions) # test_bedroc = bedroc_score(test_y, test_predictions) print("Test AUC:{}\nTest AUPRC:{}".format(validation_auc, validation_auprc, test_auc, test_auprc)) while threshold >= 0.00: validation_pred_labels = [int(round(i,2)>=threshold) for i in validation_predictions] test_pred_labels = [int(round(i,2)>=threshold) for i in test_predictions] #print(threshold) #print(test_pred_labels) val_precision = precision_score(validation_y, validation_pred_labels) val_recall = recall_score(validation_y, validation_pred_labels) val_f1score = f1_score(validation_y, validation_pred_labels) val_accuracy = accuracy_score(validation_y, validation_pred_labels) val_mcc = matthews_corrcoef(validation_y, validation_pred_labels) validation_tn, validation_fp, validation_fn, validation_tp = confusion_matrix(validation_y, validation_pred_labels).ravel() test_precision = precision_score(test_y, test_pred_labels) test_recall = recall_score(test_y, test_pred_labels) test_f1score = f1_score(test_y, test_pred_labels) test_accuracy = accuracy_score(test_y, test_pred_labels) test_mcc = matthews_corrcoef(test_y, test_pred_labels) test_tn, test_fp, test_fn, test_tp = confusion_matrix(test_y, test_pred_labels).ravel() #print(test_tn, test_fp, test_fn, test_tp) if val_f1score > best_validation_f1score_list[0]: best_validation_f1score_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp, validation_tn, validation_fn, threshold] best_test_f1score_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp, test_fp, test_tn, test_fn, threshold] if val_mcc > best_validation_mcc_list[1]: best_validation_mcc_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp, validation_tn, validation_fn, threshold] best_test_mcc_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp, test_fp, test_tn, test_fn, threshold] if val_accuracy > best_validation_accuracy_list[2]: best_validation_accuracy_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp, validation_tn, validation_fn, threshold] best_test_accuracy_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp, test_fp, test_tn, test_fn, threshold] threshold -= 0.01 print( "Test_f1score:{}\nTest_mcc:{}\nTest_accuracy:{}\nTest_precision:{}\nTest_recall:{}\nTest_tp:{}\nTest_fp:{}\nTest_tn:{}\nTest_fn:{}".format( round(best_test_mcc_list[0], 2), round(best_test_mcc_list[1], 2), round(best_test_mcc_list[2], 2), round(best_test_mcc_list[3], 2), round(best_test_mcc_list[4], 2), int(best_test_mcc_list[5]), int(best_test_mcc_list[6]), int(best_test_mcc_list[7]), int(best_test_mcc_list[8]))) best_test_threshold = round(best_test_mcc_list[-1],2) str_predictions = "" print("TestPredictions (Threshold:{})".format(best_test_threshold)) for i in range(len(test_predictions)): temp_pos_pred = round(test_predictions[i], 2) if test_y[i] == 1 and temp_pos_pred >= best_test_threshold: str_predictions += "{},{},{}\t".format(test_comp_name[i],"TP","ACT") elif test_y[i] == 1 and temp_pos_pred < best_test_threshold: str_predictions += "{},{},{}\t".format(test_comp_name[i], "FN", "ACT") elif test_y[i] == 0 and temp_pos_pred < best_test_threshold: str_predictions += "{},{},{}\t".format(test_comp_name[i], "TN", "INACT") elif test_y[i] == 0 and temp_pos_pred >= best_test_threshold: str_predictions += "{},{},{}\t".format(test_comp_name[i], "FP", "INACT") print(str_predictions)
def entropy(self, dist): return -sum([p * np.log(p) for p in dist if p > 0]) if __name__ == '__main__': config = tf.compat.v1.ConfigProto(device_count={'GPU': 1}, intra_op_parallelism_threads=1, allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.6 session = tf.compat.v1.Session(config=config) dm, y_oracle = init_dm() cnn_model = CNNModel(dm.unl.get_feature_shape(), NUM_CLASSES, epochs=5) env = Environment.create( environment=CustomEnvironment, max_episode_timesteps=100, dm=dm, session=session, cnn=cnn_model, ) agent = Agent.create( agent='dqn', states=env.states(), actions=env.actions(), batch_size=1, learning_rate=1e-3, memory=10000,
loss_name = loss.__class__.__name__ print(f"Loss: {loss_name}\n") for noise_value in noise_values: # RUN Experiments name = f'CNN_{loss_name}_{tp_noise}_{noise_value}' print(f"Training {name} with noise of type {tp_noise} and probability {noise_value}...") # data preparation dataset = FashionMnistHandler(data_dir, False) dataset.load() train_loader, val_loader, test_loader = dataset.get_noisy_loaders(p_noise=noise_value, type_noise=tp_noise, val_size=1 / 6, train_batch_size=batch_size, val_batch_size=128, test_batch_size=128) # model, optimizer, summary model = CNNModel() optimizer = torch.optim.Adam(model.parameters(), lr=lr) summ = Summary(name, type_noise=tp_noise, noise_rate=noise_value) solver = Solver(name, PROJECT_DIR, batch_model_dir, batch_summaries_dir, model, optimizer, loss, summ, train_loader, val_loader, test_loader) solver.pretrain() solver.train(loss) print(f"Completed training...")
logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-2s %(levelname)-5s %(message)s', datefmt='%y-%m-%d %H:%M', filemode='w') action_space = 15 env = WechatJumpEnv(action_space) gamma = 0.9 epsilon = .95 epsilon_min = 0.01 epsilon_decay = 0.995 learning_rate = 0.01 max_memory = 3000 tau = .125 model = CNNModel((8, 4, 3), (80, 80, 60), (100, 180, 1), action_space).build_model() target_model = CNNModel((8, 4, 3), (80, 80, 60), (100, 180, 1), action_space).build_model() dqn_agent = DQN(env, model, target_model, epsilon, epsilon_decay, epsilon_min, gamma, tau, max_memory) env.reset() for eposide in range(3000): cur_state = env.resize_input(env.screen_shot()) action = dqn_agent.choose_act(cur_state) new_state, reward, done = env.step(action) dqn_agent.remember(cur_state, action, reward, new_state, done) dqn_agent.replay() dqn_agent.target_train() # iterates target model cur_state = new_state
dtype = torch.float32 device = torch.device("cuda:0") device2 = torch.device("cuda:1") seed = 2018 np.random.seed(seed=seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) cudnn.benchmark = True sub_x, test_data, test_label = get_data() # oracle_model = Substitute(model=LeNet(), save_path='model/lenet.t7', device=device) oracle_model = Oracle(model=Rh(num_layers=1, kernel_size=7, n=100000),save_path='model/checkpoints_7_1/ckpt_100000.t7',\ svm_path='model/checkpoints_svm_7_1/svm.pkl', device=device) if sys.argv[1] == '0' or 'cnn': sub = CNNModel() else: sub = LinearModel() print(sys.argv[1]) substitute_model = Substitute(model=sub, device=device2) MNIST_bbox_sub(param=param, oracle_model=oracle_model, substitute_model=substitute_model, \ x_sub=sub_x, test_data=test_data, test_label=test_label, aug_epoch=param['data_aug'],\ samples_max=12800, n_epoch=param['nb_epochs'], fixed_lambda=param['lambda']) print('\n\nFinal results:') print('Oracle model evaluation on clean data #%d:' % (test_data.size(0))) oracle_model.eval(x=test_data, y=test_label, batch_size=10) print('Substitute model evaluation on clean data: #%d:' % (test_data.size(0))) substitute_model.eval(x=test_data, y=test_label, batch_size=512)
def main(args): print(args) startime = time.time() os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true' # Set hyper-parameters. batch_size = 128 epochs = 100 maxlen = 300 model_path = 'models/model_{}.h5' num_words = 40000 num_label = 2 # Data loading. print(return_time(startime), "1. Loading data ...") x, y = load_dataset('data/amazon_reviews_multilingual_JP_v1_00.tsv') # pre-processing. print(return_time(startime), "2. Preprocessing dataset ...") x = preprocess_dataset(x) x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42) vocab = build_vocabulary(x_train, num_words) x_train = vocab.texts_to_sequences(x_train) x_test = vocab.texts_to_sequences(x_test) x_train = pad_sequences(x_train, maxlen=maxlen, truncating='post') x_test = pad_sequences(x_test, maxlen=maxlen, truncating='post') # Preparing word embedding. if args.loadwv: print(return_time(startime), "3. Loading word embedding ...") wv_path = 'data/wv_{0}_{1}.npy'.format(maxlen, num_words) if os.path.exists(wv_path): wv = np.load(wv_path) print(return_time(startime), "Loaded word embedding successfully!") else: print(return_time(startime), "Word embedding file doesn't exist") exit() else: print(return_time(startime), "3. Preparing word embedding ...") wv = load_fasttext('data/cc.ja.300.vec.gz') wv = filter_embeddings(wv, vocab.word_index, num_words) # Saving word embedding. if args.savewv: wv_path = 'data/wv_{0}_{1}.npy'.format(maxlen, num_words) np.save(wv_path, wv) print(return_time(startime), "Saved word embedding successfully!", wv_path) # Build models. models = [ RNNModel(num_words, num_label, embeddings=None).build(), LSTMModel(num_words, num_label, embeddings=None).build(), CNNModel(num_words, num_label, embeddings=None).build(), RNNModel(num_words, num_label, embeddings=wv).build(), LSTMModel(num_words, num_label, embeddings=wv).build(), CNNModel(num_words, num_label, embeddings=wv).build(), CNNModel(num_words, num_label, embeddings=wv, trainable=False).build() ] model_names = [ "RNN-None", "LSTM-None", "CNN-None", "RNN-wv", "LSTM-wv", "CNN-wv", "CNN-wv-notrain" ] print(return_time(startime), "4. Start training ...") for i, model in enumerate(models): print("***********************************") print(return_time(startime), "Model:", model_names[i]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc']) # Preparing callbacks. callbacks = [ EarlyStopping(patience=3), ModelCheckpoint(model_path.format(model_names[i]), save_best_only=True) ] # Train the model. model.fit(x=x_train, y=y_train, batch_size=batch_size, epochs=epochs, validation_split=0.2, callbacks=callbacks, shuffle=True) # Inference. model = load_model(model_path.format(model_names[i])) api = InferenceAPI(model, vocab, preprocess_dataset) y_pred = api.predict_from_sequences(x_test) print('precision: {:.4f}'.format(precision_score(y_test, y_pred, average='binary'))) print('recall : {:.4f}'.format(recall_score(y_test, y_pred, average='binary'))) print('f1 : {:.4f}'.format(f1_score(y_test, y_pred, average='binary')))