def analyze(self, parses): lf_names = [] L = lil_matrix((len(self.Cs[1]), len(parses))) for j, p in enumerate(parses): lf_names.append(p.function.__name__) for i, c in enumerate(self.Cs[1]): L[i,j] = p.function(c) return lf_summary(L.tocsr(), Y=self.Ys[1], lf_names=lf_names)
def analysis_of_weak_labeling( data, true_labels, labeling_functions, labeling_function_names, generator_labeling_functions, ): """Displays the summary of labeling functions. Args: data (pandas df) : the text to be labeled with. true_labels (pandas series) : the current labels of the data. labeling_functions (list) : a list of labeling functions to be performed on the text. labeling_function_names (list) : a list of the name of the labeling functions to be performed on the text. generator_labeling_functions (list) : a list of labeling to be performed on the text. Returns: Displays a summary of the labeling function's success on the text. """ SummaryWriter() labeling_function_matrix = make_learning_function_matrix(data, labeling_functions) generator_labeling_function_matrix = np.empty((len(data), 1)) names = [] for labeling_function in generator_labeling_functions: new_matrix, new_names = make_large_learning_function_matrix( data, labeling_function ) generator_labeling_function_matrix = np.hstack( (generator_labeling_function_matrix, new_matrix) ) names = names + new_names concat_matrix = np.hstack( (labeling_function_matrix, generator_labeling_function_matrix[:, 1:]) ) true_labels = np.array(true_labels) print( lf_summary( sparse.csr_matrix(concat_matrix), Y=true_labels, lf_names=labeling_function_names + names, ) )
def train_label_model( self, collated_labels: np.ndarray, descriptions: Optional[List[str]], train_data_np: Optional[np.ndarray], ): sparse_labels = sparse.csr_matrix(collated_labels) if descriptions is not None: descriptions = [(i, desc) for i, desc in enumerate(descriptions)] logger.warn(f'labeling function order: {descriptions}') logger.warn(lf_summary(sparse_labels)) self.label_model.train_model( sparse_labels, n_epochs=self.num_epochs, log_train_every=self.log_train_every, Y_dev=train_data_np, )
def eval_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 1000 # using get_frm_output_size() L, Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y=Y["dev"])) # End Model # Create datasets and dataloaders dev, test = load_dataset_eval(args, Y["dev"], Y["test"]) data_loader = get_data_loader_eval(dev, test, args.batch_size, args.num_workers) #print(len(data_loader["train"])) # 18850 / batch_size #print(len(data_loader["dev"])) # 1500 / batch_size #print(len(data_loader["test"])) # 1000 / batch_size #import ipdb; ipdb.set_trace() # load model model = load_model_snapshot(args, args.pretrained_model_path) #import ipdb; ipdb.set_trace() # evaluate end model model.eval() print('Dev Set Performance:') model.score( data_loader["dev"], verbose=True, metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg']) print('\n Test Set Performance:') model.score( data_loader["test"], verbose=True, metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])
def getTrainedModel1(self): # We build a matrix of LF votes for each comment ticket LF_matrix = self.make_Ls_matrix(self.LF_set['comments'], self.LFs) # Get true labels for LF set Y_LF_set = np.array(self.LF_set['resolution']) display( lf_summary(sparse.csr_matrix(LF_matrix), Y=Y_LF_set, lf_names=self.LF_names.values())) print("label coverage: " + label_coverage(LF_matrix)) mv = MajorityLabelVoter() Y_train_majority_votes = mv.predict(LF_matrix) print("classification report:\n" + classification_report(Y_LF_set, Y_train_majority_votes)) Ls_train = self.make_Ls_matrix(self.train, self.LFs) # You can tune the learning rate and class balance. model = LabelModel(k=2, seed=123) trainer = model.train_model(Ls_train, n_epochs=2000, print_every=1000, lr=0.0001, class_balance=np.array([0.2, 0.8])) Y_train = model.predict(Ls_train) + Y_LF_set print('Trained Label Model Metrics:') scores = model.score((Ls_train[1], Y_train[1]), metric=['accuracy', 'precision', 'recall', 'f1']) print(scores) return trainer, Y_train
label_matricies['test'] .sort_values("candidate_id") .drop("candidate_id", axis=1) .to_coo() .toarray() .astype(int) ) # In[14]: lf_summary( sparse.coo_matrix( correct_L ) .tocsr(), lf_names=lf_names ) # In[15]: lf_summary( sparse.coo_matrix( correct_L_dev ) .tocsr(), lf_names=lf_names, Y=candidate_dfs['dev'].curated_cbg.apply(lambda x: 1 if x> 0 else 2)
def train_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 1000 # using get_frm_output_size() L,Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y = Y["dev"])) # training label model label_model = LabelModel(k=num_classes, seed=123) label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50) # evaluating label model print('Trained Label Model Metrics:') label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1']) # comparison with majority vote of LFs mv = MajorityLabelVoter(seed=123) print('Majority Label Voter Metrics:') mv.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1']) Ytrain_p = label_model.predict_proba(L["train"]) #print(Ytrain_ps.shape) #(377*50,2) #Ydev_p = label_model.predict_proba(L["dev"]) # test models #label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1']) # End Model # Create datasets and dataloaders train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"]) data_loader = get_data_loader(train, dev, test, args.batch_size, args.num_workers) #print(len(data_loader["train"])) # 18850 / batch_size #print(len(data_loader["dev"])) # 1500 / batch_size #print(len(data_loader["test"])) # 1000 / batch_size #import ipdb; ipdb.set_trace() # Define input encoder cnn_encoder = FrameEncoderOC if(torch.cuda.is_available()): device = 'cuda' else: device = 'cpu' #import ipdb; ipdb.set_trace() # Define LSTM module lstm_module = LSTMModule( encode_dim, hidden_size, bidirectional=False, verbose=False, lstm_reduction="attention", encoder_class=cnn_encoder, ) # Define end model end_model = EndModel( input_module=lstm_module, layer_out_dims=[hidden_size, num_classes], optimizer="adam", #use_cuda=cuda, batchnorm=True, seed=123, verbose=False, device = device, ) #print('Training model') #tic = time.time() dropout = 0.4 # Train end model end_model.train_model( train_data=data_loader["train"], valid_data=data_loader["dev"], l2=args.weight_decay, lr=args.lr, n_epochs=args.n_epochs, log_train_every=1, verbose=True, progress_bar = True, loss_weights = [0.45,0.55], batchnorm = 'True', input_dropout = dropout, middle_dropout = dropout, #validation_metric='f1', ) #print('Time taken for training:') #print(time.time() - tic) # evaluate end model end_model.score(data_loader["dev"], verbose=True, metric=['accuracy','precision', 'recall', 'f1'])
def train_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 108 # using get_frm_output_size() if (torch.cuda.is_available()): device = torch.device('cuda:0') #device = 'cuda' else: device = 'cpu' #print(device) L, Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y=Y["dev"])) # majority vote of LFs mv = MajorityLabelVoter(seed=123) print('Majority Label Voter Metrics:') mv.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) # training label model - no temporal modelling label_model = LabelModel(k=num_classes, seed=123) label_model.train_model(L["train"], Y["dev"], n_epochs=500, log_train_every=50) # evaluating label model print('Trained Label Model Metrics:') label_model.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) # training label model without temporal modelling # naive model #print(L["train"].todense().shape) # (18850,5) #print(L["dev"].todense().shape) # (1500,5) #print(Y["dev"].shape) # (1500,) m_per_task = L["train"].todense().shape[1] # 5 MRI_data_naive = { 'Li_train': torch.FloatTensor(np.array(L["train"].todense().astype('int_'))), 'Li_dev': torch.FloatTensor(np.array(L["dev"].todense())), 'R_dev': Y["dev"] } MRI_data_naive['class_balance'] = torch.FloatTensor([0.5, 0.5]).to(device) # training naive model naive_model = DPLabelModel( m=m_per_task, T=1, edges=[], coverage_sets=[[ 0, ]] * m_per_task, mu_sharing=[[ i, ] for i in range(m_per_task)], phi_sharing=[], device=device, #class_balance=MRI_data_naive['class_balance'], seed=0) optimize(naive_model, L_hat=MRI_data_naive['Li_train'], num_iter=300, lr=1e-3, momentum=0.8, clamp=True, seed=0) # evaluating naive model R_pred = naive_model.predict(MRI_data_naive['Li_dev']).data.numpy() R_pred = 2 - R_pred #print(R_pred) #print(MRI_data_naive['R_dev']) for metric in ['accuracy', 'f1', 'recall', 'precision']: score = metric_score(MRI_data_naive['R_dev'], R_pred, metric) print(f"{metric.capitalize()}: {score:.3f}") # training label model with temporal modelling # reshaping dataset num_frames = 50 n_patients_train = round(L["train"].todense().shape[0] / num_frames) #(377) n_patients_dev = round(L["dev"].todense().shape[0] / num_frames) #(30) Ltrain = np.reshape(np.array(L["train"].todense()), (n_patients_train, num_frames, -1)) Ldev = np.reshape(np.array(L["dev"].todense()), (n_patients_dev, num_frames, -1)) Ydev = np.reshape(Y["dev"], (n_patients_dev, num_frames)) # print(Ltrain.shape) # (377,50,5) #print(Ldev.shape) # (30,50,5) #print(Ydev.shape) # (30,50) # subsampling # selecting frames 3,13,23,33,43 indices = np.linspace(2, 42, 5).astype(int) m_per_task = 5 T = 5 Ltrain_small = Ltrain[:, indices, :] # shape (377,5,5) Ldev_small = Ldev[:, indices, :] # shape (30,5,5) Ydev_small = Ydev[:, indices] # shape (30,5) Ltrain_small = np.reshape( Ltrain_small, ((n_patients_train * T), m_per_task)) # shape (1885,5) Ldev_small = np.reshape( Ldev_small, ((n_patients_dev * T), m_per_task)) # shape (150,5) Ydev_small = np.reshape(Ydev_small, ((n_patients_dev * T), )) # shape (150,) MRI_data_temporal = { 'Li_train': torch.LongTensor(Ltrain_small).view(n_patients_train, (m_per_task * T)), 'Li_dev': torch.LongTensor(Ldev_small).view(n_patients_dev, (m_per_task * T)), 'R_dev': torch.LongTensor(Ydev_small)[::T] * (2**T - 1), 'm': m_per_task * T, 'T': T } MRI_data_temporal['class_balance'] = normalize( (MRI_data_temporal['R_dev'].unsqueeze(1) == torch.arange( 2**T, device=device).unsqueeze(0)).sum(0).float(), dim=0, p=1) max_seed = 10 temporal_models = [ None, ] * max_seed for seed in range(max_seed): markov_model = DPLabelModel( m=m_per_task * T, T=T, edges=[(i, i + m_per_task) for i in range((T - 1) * m_per_task)], coverage_sets=[[ t, ] for t in range(T) for _ in range(m_per_task)], mu_sharing=[[t * m_per_task + i for t in range(T)] for i in range(m_per_task)], phi_sharing=[[(t * m_per_task + i, (t + 1) * m_per_task + i) for t in range(T - 1)] for i in range(m_per_task)], device=device, class_balance=MRI_data_temporal['class_balance'], seed=seed) optimize(markov_model, L_hat=MRI_data_temporal['Li_train'], num_iter=1000, lr=1e-5, momentum=0.8, clamp=True, verbose=False, seed=seed) temporal_models[seed] = markov_model for seed, model in enumerate(temporal_models): R_pred = model.predict(MRI_data_temporal['Li_dev'].cpu()) F1 = metric_score(MRI_data_temporal['R_dev'].cpu() > 0, R_pred.cpu() > 0, 'f1') accuracy = metric_score(MRI_data_temporal['R_dev'].cpu(), R_pred.cpu(), 'accuracy') print(f"seed={seed} accuracy={accuracy:.3f} F1={F1:.3f}")
label_matricies['train'].sort_values("candidate_id")[train_ids].drop( "candidate_id", axis=1).to_coo().toarray()) correct_L_dev = plusminus_to_categorical( label_matricies['dev'].sort_values("candidate_id").drop( "candidate_id", axis=1).to_coo().toarray()) correct_L_test = plusminus_to_categorical( label_matricies['test'].sort_values("candidate_id").drop( "candidate_id", axis=1).to_coo().toarray()) # In[16]: lf_summary(sparse.coo_matrix( plusminus_to_categorical( label_matricies['train'].sort_values("candidate_id").drop( "candidate_id", axis=1).to_coo().toarray())).tocsr(), lf_names=lf_names) # In[17]: lf_summary( sparse.coo_matrix( plusminus_to_categorical( label_matricies['dev'].sort_values("candidate_id").drop( "candidate_id", axis=1).to_coo().toarray())).tocsr(), lf_names=lf_names, Y=candidate_dfs['dev'].curated_dsh.apply(lambda x: 1 if x > 0 else 2)) # The two above cells contain statistics for each label function used in this notebook. The generated statistics contains information as to the polarity of a label function, how often one label function overlaps with another, how ofter one label function conflicts with another, and many much does a given label function cover. If gold standard labels are provided, then extra information such as empirical accuracy can be incorporated. Upon insepction we have a wide range of performance in terms of individual label functions. Some are very accurate and some are almost completely wrong. Nonetheless both types of functions are important because the main goal here is to provide as much evidence as possible to estimate the tinal training class.
Xs, Ys, Ls, Ds = X, Y, L, D Xs, Ys, Ls, Ds = split_data(X, Y, L, D, splits=[0.8, 0.2], stratify_by=Y, seed=123) Xs.append(X_test) Ys.append(Y_test) Ls.append(L_test) Ds.append(D_test) print(lf_summary(Ls[1], Y=Ys[1])) balance = sorted(Counter(Y_test).items()) balance2 = Counter(Y_test).values() new_balance = [] for elem in balance: new_balance.append(elem[1] / sum(balance2)) print(sorted(Counter(Y_test).items())) print(balance) print(new_balance) label_model = LabelModel(k=2, seed=123) label_model.train_model(Ls[0], class_balance=new_balance, n_epochs=500,
plt.figure(figsize=(10, 6)) plot_predictions_histogram(dev_predictions, candidate_dfs['dev'].curated_dsh.astype(int).values, title="Prediction Histogram for Dev Set") # In[23]: confusion_matrix( convert_labels(candidate_dfs['dev'].curated_dsh.values, 'onezero', 'categorical'), convert_labels(dev_predictions, 'onezero', 'categorical')) # In[25]: lf_summary(label_matricies['dev'], Y=candidate_dfs['dev'].curated_dsh.apply(lambda x: 1 if x > 0 else 2).values, lf_names=lf_names) # In[26]: plot_label_matrix_heatmap(convert_labels(label_matricies['dev'].toarray(), 'categorical', 'plusminus').T, yaxis_tick_labels=lf_names, figsize=(10, 12), font_size=10) # In[27]: output_file = "data/train_marginals.pkl" pickle.dump(label_model.predict_proba(L[:, :24]), open(output_file, "wb"))
from lfconfig import president_name_pob_lfs, TRUE from fonduer.supervision import Labeler labeler = Labeler(session, candidate_classes) labeler.apply(split=0, lfs=[president_name_pob_lfs], train=True, parallelism=PARALLEL) L_train = labeler.get_label_matrices(train_cands) L_gold_train = labeler.get_gold_labels(train_cands, annotator="gold") from metal import analysis analysis.lf_summary( L_train[0], lf_names=labeler.get_keys(), Y=L_gold_train[0].todense().reshape(-1).tolist()[0], ) from metal.label_model import LabelModel gen_model = LabelModel(k=2) gen_model.train_model(L_train[0], n_epochs=500, print_every=100) train_marginals = gen_model.predict_proba(L_train[0]) from fonduer.learning import LogisticRegression disc_model = LogisticRegression() disc_model.train((train_cands[0], F_train[0]), train_marginals, n_epochs=10, lr=0.001)
def train_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 108 # using get_frm_output_size() L, Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y=Y["dev"])) # majority vote of LFs mv = MajorityLabelVoter(seed=123) print('Majority Label Voter Metrics:') mv.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) # training label model #label_model = LabelModel(k=num_classes, seed=123) #label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50) # evaluating label model #print('Trained Label Model Metrics:') #label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1']) print('Performing Hyperparameter Search:') train_args = [L["train"], Y["dev"]] train_kwargs = {} init_args = [num_classes] init_kwargs = { "optimizer": "sgd", #"input_batchnorm": True, #"use_cuda":torch.cuda.is_available(), 'seed': 123 } search_space = { 'seed': [123], 'n_epochs': [500], 'learn_class_balance': [False, True], 'lr': { 'range': [1e-2, 1e-1], 'scale': 'log' }, 'momentum': { 'range': [0.7, 0.95], 'scale': 'log' }, #'l2':{'range': [1e-5, 1e-3], 'scale': 'log'}, 'log_train_every': [50], #'validation_metric': 'accuracy', } log_config = {"log_dir": "./run_logs", "run_name": 'oc_label_model'} max_search = 25 tuner_config = {"max_search": max_search} validation_metric = 'accuracy' # Set up logger and searcher tuner = RandomSearchTuner( LabelModel, #**log_config, #log_writer_class=TensorBoardWriter, validation_metric=validation_metric, seed=1701) disc_model = tuner.search(search_space, valid_data=(L["dev"], Y["dev"]), train_args=train_args, init_args=init_args, init_kwargs=init_kwargs, train_kwargs=train_kwargs, max_search=tuner_config["max_search"], clean_up=False) print('Trained Label Model Metrics:') disc_model.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) Ytrain_p = disc_model.predict_proba(L["train"])
def train_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 1000 # using get_frm_output_size() L,Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y = Y["dev"])) # training label model label_model = LabelModel(k=num_classes, seed=123) label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50) # evaluating label model print('Trained Label Model Metrics:') label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1']) # comparison with majority vote of LFs mv = MajorityLabelVoter(seed=123) print('Majority Label Voter Metrics:') mv.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1']) Ytrain_p = label_model.predict_proba(L["train"]) #print(Ytrain_ps.shape) #(377*50,2) #Ydev_p = label_model.predict_proba(L["dev"]) # test models #label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1']) # End Model # Create datasets and dataloaders train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"]) data_loader = get_data_loader(train, dev, test, args.batch_size, args.num_workers) #print(len(data_loader["train"])) # 18850 / batch_size #print(len(data_loader["dev"])) # 1500 / batch_size #print(len(data_loader["test"])) # 1000 / batch_size #import ipdb; ipdb.set_trace() # Define input encoder cnn_encoder = FrameEncoderOC if(torch.cuda.is_available()): device = 'cuda' else: device = 'cpu' #import ipdb; ipdb.set_trace() # Define LSTM module lstm_module = LSTMModule( encode_dim, hidden_size, bidirectional=False, verbose=False, lstm_reduction="attention", encoder_class=cnn_encoder, ) train_args = [data_loader["train"]] train_kwargs = { 'seed':args.seed, 'progress_bar':True, 'log_train_every':1} init_args = [ [hidden_size, num_classes] ] init_kwargs = { "input_module": lstm_module, "optimizer": "adam", "verbose": False, "input_batchnorm": True, "use_cuda":torch.cuda.is_available(), 'checkpoint_dir':args.checkpoint_dir, 'seed':args.seed, 'device':device} search_space = { 'n_epochs':[10], 'batchnorm':[True], 'dropout': [0.1,0.25,0.4], 'lr':{'range': [1e-3, 1e-2], 'scale': 'log'}, 'l2':{'range': [1e-5, 1e-4], 'scale': 'log'},#[ 1.21*1e-5], #'checkpoint_metric':['f1'], } log_config = { "log_dir": "./run_logs", "run_name": 'cnn_lstm_oc' } max_search = 5 tuner_config = {"max_search": max_search } validation_metric = 'accuracy' # Set up logger and searcher tuner = RandomSearchTuner(EndModel, **log_config, log_writer_class=TensorBoardWriter, validation_metric=validation_metric, seed=1701) disc_model = tuner.search( search_space, valid_data = data_loader["dev"], train_args=train_args, init_args=init_args, init_kwargs=init_kwargs, train_kwargs=train_kwargs, max_search=tuner_config["max_search"], clean_up=False, ) # evaluate end model disc_model.score(data_loader["dev"], verbose=True, metric=['accuracy','precision', 'recall', 'f1'])
def train_model(args): #global args #args = parser.parse_args() hidden_size = 128 num_classes = 2 encode_dim = 1000 # using get_frm_output_size() L, Y = load_labels(args) # Label Model # labelling functions analysis print(lf_summary(L["dev"], Y=Y["dev"])) # training label model label_model = LabelModel(k=num_classes, seed=123) label_model.train_model(L["train"], Y["dev"], n_epochs=2000, log_train_every=100) # evaluating label model print('Trained Label Model Metrics:') label_model.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) # comparison with majority vote of LFs mv = MajorityLabelVoter(seed=123) print('Majority Label Voter Metrics:') mv.score((L["dev"], Y["dev"]), metric=['accuracy', 'precision', 'recall', 'f1']) Ytrain_p = label_model.predict_proba(L["train"]) #print(Ytrain_ps.shape) #(377*50,2) #Ydev_p = label_model.predict_proba(L["dev"]) # test models #label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1']) # End Model # Create datasets and dataloaders train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"]) data_loader = get_data_loader(train, dev, test, args.batch_size, args.num_workers) #print(len(data_loader["train"])) # 18850 / batch_size #print(len(data_loader["dev"])) # 1500 / batch_size #print(len(data_loader["test"])) # 1000 / batch_size #import ipdb; ipdb.set_trace() # Define input encoder #cnn_encoder = FrameEncoderOC cnn_encoder = FrameEncoderOCDense if (torch.cuda.is_available()): device = 'cuda' else: device = 'cpu' #import ipdb; ipdb.set_trace() # Define LSTM module lstm_module = LSTMModule( encode_dim, hidden_size, bidirectional=False, verbose=False, lstm_reduction=args.lstm_reduction, encoder_class=cnn_encoder, encoder_kwargs={"requires_grad": args.requires_grad}) ''' # Define end model end_model = EndModel( input_module=lstm_module, layer_out_dims=[hidden_size, num_classes], optimizer="adam", #use_cuda=cuda, batchnorm=False, seed=args.seed, verbose=False, device = device, ) ''' init_kwargs = { "layer_out_dims": [hidden_size, num_classes], "input_module": lstm_module, "optimizer": "adam", "verbose": False, "input_batchnorm": False, "use_cuda": cuda, 'seed': args.seed, 'device': device } end_model = EndModel(**init_kwargs) if not os.path.exists(args.checkpoint_dir): os.mkdir(args.checkpoint_dir) with open(args.checkpoint_dir + '/init_kwargs.pickle', "wb") as f: pickle.dump(init_kwargs, f, protocol=pickle.HIGHEST_PROTOCOL) dropout = 0.4 # Train end model end_model.train_model( train_data=data_loader["train"], valid_data=data_loader["dev"], l2=args.weight_decay, lr=args.lr, n_epochs=args.n_epochs, log_train_every=1, verbose=True, progress_bar=True, loss_weights=[0.55, 0.45], input_dropout=0.1, middle_dropout=dropout, checkpoint_dir=args.checkpoint_dir, #writer = "json", #writer_config = { #"log_dir": args.log_dir, #"run_dir": args.run_dir, #"run_name": args.run_name, #"writer_metrics": ['accuracy','precision', 'recall', 'f1','roc-auc','ndcg'] #}, #validation_metric='f1', ) # evaluate end model print("Dev Set Performance") end_model.score( data_loader["dev"], verbose=True, metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg']) print("Test Set Performance") end_model.score( data_loader["test"], verbose=True, metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])