def analyze(self, parses):
lf_names = []
L = lil_matrix((len(self.Cs[1]), len(parses)))
for j, p in enumerate(parses):
lf_names.append(p.function.__name__)
for i, c in enumerate(self.Cs[1]):
L[i,j] = p.function(c)
return lf_summary(L.tocsr(), Y=self.Ys[1], lf_names=lf_names)
def analysis_of_weak_labeling(
data,
true_labels,
labeling_functions,
labeling_function_names,
generator_labeling_functions,
):
"""Displays the summary of labeling functions.
Args:
data (pandas df) : the text to be labeled with.
true_labels (pandas series) : the current labels of the data.
labeling_functions (list) : a list of labeling functions to be
performed on the text.
labeling_function_names (list) : a list of the name of the
labeling functions to be performed on the text.
generator_labeling_functions (list) : a list of labeling
to be performed on the text.
Returns:
Displays a summary of the labeling function's success on the
text.
"""
SummaryWriter()
labeling_function_matrix = make_learning_function_matrix(data, labeling_functions)
generator_labeling_function_matrix = np.empty((len(data), 1))
names = []
for labeling_function in generator_labeling_functions:
new_matrix, new_names = make_large_learning_function_matrix(
data, labeling_function
)
generator_labeling_function_matrix = np.hstack(
(generator_labeling_function_matrix, new_matrix)
)
names = names + new_names
concat_matrix = np.hstack(
(labeling_function_matrix, generator_labeling_function_matrix[:, 1:])
)
true_labels = np.array(true_labels)
print(
lf_summary(
sparse.csr_matrix(concat_matrix),
Y=true_labels,
lf_names=labeling_function_names + names,
)
)
def train_label_model(
self,
collated_labels: np.ndarray,
descriptions: Optional[List[str]],
train_data_np: Optional[np.ndarray],
):
sparse_labels = sparse.csr_matrix(collated_labels)
if descriptions is not None:
descriptions = [(i, desc) for i, desc in enumerate(descriptions)]
logger.warn(f'labeling function order: {descriptions}')
logger.warn(lf_summary(sparse_labels))
self.label_model.train_model(
sparse_labels,
n_epochs=self.num_epochs,
log_train_every=self.log_train_every,
Y_dev=train_data_np,
)
def eval_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 1000 # using get_frm_output_size()
L, Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y=Y["dev"]))
# End Model
# Create datasets and dataloaders
dev, test = load_dataset_eval(args, Y["dev"], Y["test"])
data_loader = get_data_loader_eval(dev, test, args.batch_size,
args.num_workers)
#print(len(data_loader["train"])) # 18850 / batch_size
#print(len(data_loader["dev"])) # 1500 / batch_size
#print(len(data_loader["test"])) # 1000 / batch_size
#import ipdb; ipdb.set_trace()
# load model
model = load_model_snapshot(args, args.pretrained_model_path)
#import ipdb; ipdb.set_trace()
# evaluate end model
model.eval()
print('Dev Set Performance:')
model.score(
data_loader["dev"],
verbose=True,
metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])
print('\n Test Set Performance:')
model.score(
data_loader["test"],
verbose=True,
metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])
def getTrainedModel1(self):
# We build a matrix of LF votes for each comment ticket
LF_matrix = self.make_Ls_matrix(self.LF_set['comments'], self.LFs)
# Get true labels for LF set
Y_LF_set = np.array(self.LF_set['resolution'])
display(
lf_summary(sparse.csr_matrix(LF_matrix),
Y=Y_LF_set,
lf_names=self.LF_names.values()))
print("label coverage: " + label_coverage(LF_matrix))
mv = MajorityLabelVoter()
Y_train_majority_votes = mv.predict(LF_matrix)
print("classification report:\n" +
classification_report(Y_LF_set, Y_train_majority_votes))
Ls_train = self.make_Ls_matrix(self.train, self.LFs)
# You can tune the learning rate and class balance.
model = LabelModel(k=2, seed=123)
trainer = model.train_model(Ls_train,
n_epochs=2000,
print_every=1000,
lr=0.0001,
class_balance=np.array([0.2, 0.8]))
Y_train = model.predict(Ls_train) + Y_LF_set
print('Trained Label Model Metrics:')
scores = model.score((Ls_train[1], Y_train[1]),
metric=['accuracy', 'precision', 'recall', 'f1'])
print(scores)
return trainer, Y_train
label_matricies['test']
.sort_values("candidate_id")
.drop("candidate_id", axis=1)
.to_coo()
.toarray()
.astype(int)
)
# In[14]:
lf_summary(
sparse.coo_matrix(
correct_L
)
.tocsr(),
lf_names=lf_names
)
# In[15]:
lf_summary(
sparse.coo_matrix(
correct_L_dev
)
.tocsr(),
lf_names=lf_names,
Y=candidate_dfs['dev'].curated_cbg.apply(lambda x: 1 if x> 0 else 2)
def train_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 1000 # using get_frm_output_size()
L,Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y = Y["dev"]))
# training label model
label_model = LabelModel(k=num_classes, seed=123)
label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50)
# evaluating label model
print('Trained Label Model Metrics:')
label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1'])
# comparison with majority vote of LFs
mv = MajorityLabelVoter(seed=123)
print('Majority Label Voter Metrics:')
mv.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1'])
Ytrain_p = label_model.predict_proba(L["train"])
#print(Ytrain_ps.shape) #(377*50,2)
#Ydev_p = label_model.predict_proba(L["dev"])
# test models
#label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1'])
# End Model
# Create datasets and dataloaders
train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"])
data_loader = get_data_loader(train, dev, test, args.batch_size, args.num_workers)
#print(len(data_loader["train"])) # 18850 / batch_size
#print(len(data_loader["dev"])) # 1500 / batch_size
#print(len(data_loader["test"])) # 1000 / batch_size
#import ipdb; ipdb.set_trace()
# Define input encoder
cnn_encoder = FrameEncoderOC
if(torch.cuda.is_available()):
device = 'cuda'
else:
device = 'cpu'
#import ipdb; ipdb.set_trace()
# Define LSTM module
lstm_module = LSTMModule(
encode_dim,
hidden_size,
bidirectional=False,
verbose=False,
lstm_reduction="attention",
encoder_class=cnn_encoder,
)
# Define end model
end_model = EndModel(
input_module=lstm_module,
layer_out_dims=[hidden_size, num_classes],
optimizer="adam",
#use_cuda=cuda,
batchnorm=True,
seed=123,
verbose=False,
device = device,
)
#print('Training model')
#tic = time.time()
dropout = 0.4
# Train end model
end_model.train_model(
train_data=data_loader["train"],
valid_data=data_loader["dev"],
l2=args.weight_decay,
lr=args.lr,
n_epochs=args.n_epochs,
log_train_every=1,
verbose=True,
progress_bar = True,
loss_weights = [0.45,0.55],
batchnorm = 'True',
input_dropout = dropout,
middle_dropout = dropout,
#validation_metric='f1',
)
#print('Time taken for training:')
#print(time.time() - tic)
# evaluate end model
end_model.score(data_loader["dev"], verbose=True, metric=['accuracy','precision', 'recall', 'f1'])
def train_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 108 # using get_frm_output_size()
if (torch.cuda.is_available()):
device = torch.device('cuda:0')
#device = 'cuda'
else:
device = 'cpu'
#print(device)
L, Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y=Y["dev"]))
# majority vote of LFs
mv = MajorityLabelVoter(seed=123)
print('Majority Label Voter Metrics:')
mv.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
# training label model - no temporal modelling
label_model = LabelModel(k=num_classes, seed=123)
label_model.train_model(L["train"],
Y["dev"],
n_epochs=500,
log_train_every=50)
# evaluating label model
print('Trained Label Model Metrics:')
label_model.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
# training label model without temporal modelling
# naive model
#print(L["train"].todense().shape) # (18850,5)
#print(L["dev"].todense().shape) # (1500,5)
#print(Y["dev"].shape) # (1500,)
m_per_task = L["train"].todense().shape[1] # 5
MRI_data_naive = {
'Li_train':
torch.FloatTensor(np.array(L["train"].todense().astype('int_'))),
'Li_dev':
torch.FloatTensor(np.array(L["dev"].todense())),
'R_dev':
Y["dev"]
}
MRI_data_naive['class_balance'] = torch.FloatTensor([0.5, 0.5]).to(device)
# training naive model
naive_model = DPLabelModel(
m=m_per_task,
T=1,
edges=[],
coverage_sets=[[
0,
]] * m_per_task,
mu_sharing=[[
i,
] for i in range(m_per_task)],
phi_sharing=[],
device=device,
#class_balance=MRI_data_naive['class_balance'],
seed=0)
optimize(naive_model,
L_hat=MRI_data_naive['Li_train'],
num_iter=300,
lr=1e-3,
momentum=0.8,
clamp=True,
seed=0)
# evaluating naive model
R_pred = naive_model.predict(MRI_data_naive['Li_dev']).data.numpy()
R_pred = 2 - R_pred
#print(R_pred)
#print(MRI_data_naive['R_dev'])
for metric in ['accuracy', 'f1', 'recall', 'precision']:
score = metric_score(MRI_data_naive['R_dev'], R_pred, metric)
print(f"{metric.capitalize()}: {score:.3f}")
# training label model with temporal modelling
# reshaping dataset
num_frames = 50
n_patients_train = round(L["train"].todense().shape[0] /
num_frames) #(377)
n_patients_dev = round(L["dev"].todense().shape[0] / num_frames) #(30)
Ltrain = np.reshape(np.array(L["train"].todense()),
(n_patients_train, num_frames, -1))
Ldev = np.reshape(np.array(L["dev"].todense()),
(n_patients_dev, num_frames, -1))
Ydev = np.reshape(Y["dev"], (n_patients_dev, num_frames))
# print(Ltrain.shape) # (377,50,5)
#print(Ldev.shape) # (30,50,5)
#print(Ydev.shape) # (30,50)
# subsampling
# selecting frames 3,13,23,33,43
indices = np.linspace(2, 42, 5).astype(int)
m_per_task = 5
T = 5
Ltrain_small = Ltrain[:, indices, :] # shape (377,5,5)
Ldev_small = Ldev[:, indices, :] # shape (30,5,5)
Ydev_small = Ydev[:, indices] # shape (30,5)
Ltrain_small = np.reshape(
Ltrain_small, ((n_patients_train * T), m_per_task)) # shape (1885,5)
Ldev_small = np.reshape(
Ldev_small, ((n_patients_dev * T), m_per_task)) # shape (150,5)
Ydev_small = np.reshape(Ydev_small,
((n_patients_dev * T), )) # shape (150,)
MRI_data_temporal = {
'Li_train':
torch.LongTensor(Ltrain_small).view(n_patients_train,
(m_per_task * T)),
'Li_dev':
torch.LongTensor(Ldev_small).view(n_patients_dev, (m_per_task * T)),
'R_dev':
torch.LongTensor(Ydev_small)[::T] * (2**T - 1),
'm':
m_per_task * T,
'T':
T
}
MRI_data_temporal['class_balance'] = normalize(
(MRI_data_temporal['R_dev'].unsqueeze(1) == torch.arange(
2**T, device=device).unsqueeze(0)).sum(0).float(),
dim=0,
p=1)
max_seed = 10
temporal_models = [
None,
] * max_seed
for seed in range(max_seed):
markov_model = DPLabelModel(
m=m_per_task * T,
T=T,
edges=[(i, i + m_per_task) for i in range((T - 1) * m_per_task)],
coverage_sets=[[
t,
] for t in range(T) for _ in range(m_per_task)],
mu_sharing=[[t * m_per_task + i for t in range(T)]
for i in range(m_per_task)],
phi_sharing=[[(t * m_per_task + i, (t + 1) * m_per_task + i)
for t in range(T - 1)] for i in range(m_per_task)],
device=device,
class_balance=MRI_data_temporal['class_balance'],
seed=seed)
optimize(markov_model,
L_hat=MRI_data_temporal['Li_train'],
num_iter=1000,
lr=1e-5,
momentum=0.8,
clamp=True,
verbose=False,
seed=seed)
temporal_models[seed] = markov_model
for seed, model in enumerate(temporal_models):
R_pred = model.predict(MRI_data_temporal['Li_dev'].cpu())
F1 = metric_score(MRI_data_temporal['R_dev'].cpu() > 0,
R_pred.cpu() > 0, 'f1')
accuracy = metric_score(MRI_data_temporal['R_dev'].cpu(), R_pred.cpu(),
'accuracy')
print(f"seed={seed} accuracy={accuracy:.3f} F1={F1:.3f}")
label_matricies['train'].sort_values("candidate_id")[train_ids].drop(
"candidate_id", axis=1).to_coo().toarray())
correct_L_dev = plusminus_to_categorical(
label_matricies['dev'].sort_values("candidate_id").drop(
"candidate_id", axis=1).to_coo().toarray())
correct_L_test = plusminus_to_categorical(
label_matricies['test'].sort_values("candidate_id").drop(
"candidate_id", axis=1).to_coo().toarray())
# In[16]:
lf_summary(sparse.coo_matrix(
plusminus_to_categorical(
label_matricies['train'].sort_values("candidate_id").drop(
"candidate_id", axis=1).to_coo().toarray())).tocsr(),
lf_names=lf_names)
# In[17]:
lf_summary(
sparse.coo_matrix(
plusminus_to_categorical(
label_matricies['dev'].sort_values("candidate_id").drop(
"candidate_id", axis=1).to_coo().toarray())).tocsr(),
lf_names=lf_names,
Y=candidate_dfs['dev'].curated_dsh.apply(lambda x: 1 if x > 0 else 2))
# The two above cells contain statistics for each label function used in this notebook. The generated statistics contains information as to the polarity of a label function, how often one label function overlaps with another, how ofter one label function conflicts with another, and many much does a given label function cover. If gold standard labels are provided, then extra information such as empirical accuracy can be incorporated. Upon insepction we have a wide range of performance in terms of individual label functions. Some are very accurate and some are almost completely wrong. Nonetheless both types of functions are important because the main goal here is to provide as much evidence as possible to estimate the tinal training class.
Xs, Ys, Ls, Ds = X, Y, L, D
Xs, Ys, Ls, Ds = split_data(X,
Y,
L,
D,
splits=[0.8, 0.2],
stratify_by=Y,
seed=123)
Xs.append(X_test)
Ys.append(Y_test)
Ls.append(L_test)
Ds.append(D_test)
print(lf_summary(Ls[1], Y=Ys[1]))
balance = sorted(Counter(Y_test).items())
balance2 = Counter(Y_test).values()
new_balance = []
for elem in balance:
new_balance.append(elem[1] / sum(balance2))
print(sorted(Counter(Y_test).items()))
print(balance)
print(new_balance)
label_model = LabelModel(k=2, seed=123)
label_model.train_model(Ls[0],
class_balance=new_balance,
n_epochs=500,
plt.figure(figsize=(10, 6))
plot_predictions_histogram(dev_predictions,
candidate_dfs['dev'].curated_dsh.astype(int).values,
title="Prediction Histogram for Dev Set")
# In[23]:
confusion_matrix(
convert_labels(candidate_dfs['dev'].curated_dsh.values, 'onezero',
'categorical'),
convert_labels(dev_predictions, 'onezero', 'categorical'))
# In[25]:
lf_summary(label_matricies['dev'],
Y=candidate_dfs['dev'].curated_dsh.apply(lambda x: 1
if x > 0 else 2).values,
lf_names=lf_names)
# In[26]:
plot_label_matrix_heatmap(convert_labels(label_matricies['dev'].toarray(),
'categorical', 'plusminus').T,
yaxis_tick_labels=lf_names,
figsize=(10, 12),
font_size=10)
# In[27]:
output_file = "data/train_marginals.pkl"
pickle.dump(label_model.predict_proba(L[:, :24]), open(output_file, "wb"))
from lfconfig import president_name_pob_lfs, TRUE
from fonduer.supervision import Labeler
labeler = Labeler(session, candidate_classes)
labeler.apply(split=0, lfs=[president_name_pob_lfs], train=True, parallelism=PARALLEL)
L_train = labeler.get_label_matrices(train_cands)
L_gold_train = labeler.get_gold_labels(train_cands, annotator="gold")
from metal import analysis
analysis.lf_summary(
L_train[0],
lf_names=labeler.get_keys(),
Y=L_gold_train[0].todense().reshape(-1).tolist()[0],
)
from metal.label_model import LabelModel
gen_model = LabelModel(k=2)
gen_model.train_model(L_train[0], n_epochs=500, print_every=100)
train_marginals = gen_model.predict_proba(L_train[0])
from fonduer.learning import LogisticRegression
disc_model = LogisticRegression()
disc_model.train((train_cands[0], F_train[0]), train_marginals, n_epochs=10, lr=0.001)
def train_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 108 # using get_frm_output_size()
L, Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y=Y["dev"]))
# majority vote of LFs
mv = MajorityLabelVoter(seed=123)
print('Majority Label Voter Metrics:')
mv.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
# training label model
#label_model = LabelModel(k=num_classes, seed=123)
#label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50)
# evaluating label model
#print('Trained Label Model Metrics:')
#label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1'])
print('Performing Hyperparameter Search:')
train_args = [L["train"], Y["dev"]]
train_kwargs = {}
init_args = [num_classes]
init_kwargs = {
"optimizer": "sgd",
#"input_batchnorm": True,
#"use_cuda":torch.cuda.is_available(),
'seed': 123
}
search_space = {
'seed': [123],
'n_epochs': [500],
'learn_class_balance': [False, True],
'lr': {
'range': [1e-2, 1e-1],
'scale': 'log'
},
'momentum': {
'range': [0.7, 0.95],
'scale': 'log'
},
#'l2':{'range': [1e-5, 1e-3], 'scale': 'log'},
'log_train_every': [50],
#'validation_metric': 'accuracy',
}
log_config = {"log_dir": "./run_logs", "run_name": 'oc_label_model'}
max_search = 25
tuner_config = {"max_search": max_search}
validation_metric = 'accuracy'
# Set up logger and searcher
tuner = RandomSearchTuner(
LabelModel,
#**log_config,
#log_writer_class=TensorBoardWriter,
validation_metric=validation_metric,
seed=1701)
disc_model = tuner.search(search_space,
valid_data=(L["dev"], Y["dev"]),
train_args=train_args,
init_args=init_args,
init_kwargs=init_kwargs,
train_kwargs=train_kwargs,
max_search=tuner_config["max_search"],
clean_up=False)
print('Trained Label Model Metrics:')
disc_model.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
Ytrain_p = disc_model.predict_proba(L["train"])
def train_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 1000 # using get_frm_output_size()
L,Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y = Y["dev"]))
# training label model
label_model = LabelModel(k=num_classes, seed=123)
label_model.train_model(L["train"], Y["dev"], n_epochs = 500, log_train_every = 50)
# evaluating label model
print('Trained Label Model Metrics:')
label_model.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1'])
# comparison with majority vote of LFs
mv = MajorityLabelVoter(seed=123)
print('Majority Label Voter Metrics:')
mv.score((L["dev"], Y["dev"]), metric=['accuracy','precision', 'recall', 'f1'])
Ytrain_p = label_model.predict_proba(L["train"])
#print(Ytrain_ps.shape) #(377*50,2)
#Ydev_p = label_model.predict_proba(L["dev"])
# test models
#label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1'])
# End Model
# Create datasets and dataloaders
train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"])
data_loader = get_data_loader(train, dev, test, args.batch_size, args.num_workers)
#print(len(data_loader["train"])) # 18850 / batch_size
#print(len(data_loader["dev"])) # 1500 / batch_size
#print(len(data_loader["test"])) # 1000 / batch_size
#import ipdb; ipdb.set_trace()
# Define input encoder
cnn_encoder = FrameEncoderOC
if(torch.cuda.is_available()):
device = 'cuda'
else:
device = 'cpu'
#import ipdb; ipdb.set_trace()
# Define LSTM module
lstm_module = LSTMModule(
encode_dim,
hidden_size,
bidirectional=False,
verbose=False,
lstm_reduction="attention",
encoder_class=cnn_encoder,
)
train_args = [data_loader["train"]]
train_kwargs = {
'seed':args.seed,
'progress_bar':True,
'log_train_every':1}
init_args = [
[hidden_size, num_classes]
]
init_kwargs = {
"input_module": lstm_module,
"optimizer": "adam",
"verbose": False,
"input_batchnorm": True,
"use_cuda":torch.cuda.is_available(),
'checkpoint_dir':args.checkpoint_dir,
'seed':args.seed,
'device':device}
search_space = {
'n_epochs':[10],
'batchnorm':[True],
'dropout': [0.1,0.25,0.4],
'lr':{'range': [1e-3, 1e-2], 'scale': 'log'},
'l2':{'range': [1e-5, 1e-4], 'scale': 'log'},#[ 1.21*1e-5],
#'checkpoint_metric':['f1'],
}
log_config = {
"log_dir": "./run_logs",
"run_name": 'cnn_lstm_oc'
}
max_search = 5
tuner_config = {"max_search": max_search }
validation_metric = 'accuracy'
# Set up logger and searcher
tuner = RandomSearchTuner(EndModel,
**log_config,
log_writer_class=TensorBoardWriter,
validation_metric=validation_metric,
seed=1701)
disc_model = tuner.search(
search_space,
valid_data = data_loader["dev"],
train_args=train_args,
init_args=init_args,
init_kwargs=init_kwargs,
train_kwargs=train_kwargs,
max_search=tuner_config["max_search"],
clean_up=False,
)
# evaluate end model
disc_model.score(data_loader["dev"], verbose=True, metric=['accuracy','precision', 'recall', 'f1'])
def train_model(args):
#global args
#args = parser.parse_args()
hidden_size = 128
num_classes = 2
encode_dim = 1000 # using get_frm_output_size()
L, Y = load_labels(args)
# Label Model
# labelling functions analysis
print(lf_summary(L["dev"], Y=Y["dev"]))
# training label model
label_model = LabelModel(k=num_classes, seed=123)
label_model.train_model(L["train"],
Y["dev"],
n_epochs=2000,
log_train_every=100)
# evaluating label model
print('Trained Label Model Metrics:')
label_model.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
# comparison with majority vote of LFs
mv = MajorityLabelVoter(seed=123)
print('Majority Label Voter Metrics:')
mv.score((L["dev"], Y["dev"]),
metric=['accuracy', 'precision', 'recall', 'f1'])
Ytrain_p = label_model.predict_proba(L["train"])
#print(Ytrain_ps.shape) #(377*50,2)
#Ydev_p = label_model.predict_proba(L["dev"])
# test models
#label_model.score((Ltest,Ytest), metric=['accuracy','precision', 'recall', 'f1'])
# End Model
# Create datasets and dataloaders
train, dev, test = load_dataset(args, Ytrain_p, Y["dev"], Y["test"])
data_loader = get_data_loader(train, dev, test, args.batch_size,
args.num_workers)
#print(len(data_loader["train"])) # 18850 / batch_size
#print(len(data_loader["dev"])) # 1500 / batch_size
#print(len(data_loader["test"])) # 1000 / batch_size
#import ipdb; ipdb.set_trace()
# Define input encoder
#cnn_encoder = FrameEncoderOC
cnn_encoder = FrameEncoderOCDense
if (torch.cuda.is_available()):
device = 'cuda'
else:
device = 'cpu'
#import ipdb; ipdb.set_trace()
# Define LSTM module
lstm_module = LSTMModule(
encode_dim,
hidden_size,
bidirectional=False,
verbose=False,
lstm_reduction=args.lstm_reduction,
encoder_class=cnn_encoder,
encoder_kwargs={"requires_grad": args.requires_grad})
'''
# Define end model
end_model = EndModel(
input_module=lstm_module,
layer_out_dims=[hidden_size, num_classes],
optimizer="adam",
#use_cuda=cuda,
batchnorm=False,
seed=args.seed,
verbose=False,
device = device,
)
'''
init_kwargs = {
"layer_out_dims": [hidden_size, num_classes],
"input_module": lstm_module,
"optimizer": "adam",
"verbose": False,
"input_batchnorm": False,
"use_cuda": cuda,
'seed': args.seed,
'device': device
}
end_model = EndModel(**init_kwargs)
if not os.path.exists(args.checkpoint_dir):
os.mkdir(args.checkpoint_dir)
with open(args.checkpoint_dir + '/init_kwargs.pickle', "wb") as f:
pickle.dump(init_kwargs, f, protocol=pickle.HIGHEST_PROTOCOL)
dropout = 0.4
# Train end model
end_model.train_model(
train_data=data_loader["train"],
valid_data=data_loader["dev"],
l2=args.weight_decay,
lr=args.lr,
n_epochs=args.n_epochs,
log_train_every=1,
verbose=True,
progress_bar=True,
loss_weights=[0.55, 0.45],
input_dropout=0.1,
middle_dropout=dropout,
checkpoint_dir=args.checkpoint_dir,
#writer = "json",
#writer_config = {
#"log_dir": args.log_dir,
#"run_dir": args.run_dir,
#"run_name": args.run_name,
#"writer_metrics": ['accuracy','precision', 'recall', 'f1','roc-auc','ndcg']
#},
#validation_metric='f1',
)
# evaluate end model
print("Dev Set Performance")
end_model.score(
data_loader["dev"],
verbose=True,
metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])
print("Test Set Performance")
end_model.score(
data_loader["test"],
verbose=True,
metric=['accuracy', 'precision', 'recall', 'f1', 'roc-auc', 'ndcg'])
