def run_model(model_type,
in_notebook=False,
is_verbose=True,
return_model=False):
if model_type == 'LR':
print(
"\nPredicting Speaker Stance - Multi Label Logistic Regression Baseline Model "
)
model = baseline.Model()
elif model_type == 'MultiClassLR':
print(
"\nPredicting Speaker Stance - Multi Class Logistic Regression Baseline Model "
)
model = multiclassbaseline.Model()
elif model_type == 'MTNN':
print("\nPredicting Speaker Stance - Multi Task Model ")
model = MultiTaskNN.My_Model(is_verbose=is_verbose)
elif model_type == 'MLP':
print("\nPredicting Speaker Stance - Multi Layer Perceptron Model ")
model = MultiLayerPercep.Model(is_verbose=is_verbose)
elif model_type == 'FastText':
print("\nPredicting Speaker Stance - FastText Model ")
model = FastText.My_Model(is_verbose=is_verbose)
elif model_type == 'MultiClassFastText':
print("\nPredicting Speaker Stance - FastText Model ")
model = FastTextMultiClass.My_Model(is_verbose=is_verbose)
elif model_type == 'FastTextMT':
print("\nPredicting Speaker Stance - FastText Multi Task Model ")
model = FastTextMultiTask.My_Model(is_verbose=is_verbose)
elif model_type == 'FastTextAux':
print("\nPredicting Speaker Stance - FastText Aux Task Model ")
model = FastTextAuxTask.My_Model(is_verbose=is_verbose)
else:
raise ValueError(
'Unknown Model Type: {} not supported option'.format(model_type))
print("Loading Data")
if in_notebook is True:
data = Dataset(in_notebook=True)
else:
data = Dataset()
print("Training Model")
model.train(data.train_set())
y, y_pred = model.test(data.test_set())
if return_model is True:
return model, y, y_pred
return y, y_pred
def run_model(test_language, evaluation_split, detailed_report):
""" Trains the CWI model in all languages but one. Tests on all datasets of
a particular language. Reports results.
Args:
test_language: The language of the dataset to use for testing.
evaluation_split: The split of the data to use for
evaluating the performance of the model (dev or
test).
detailed_report: Whether to display a detailed report or just overall
score.
"""
# collect the training data for all the languages but one
train_data = []
for language, datasets_names in datasets_per_language.items():
if language != test_language:
for dataset_name in datasets_names:
data = Dataset(language, dataset_name)
lang_train_set = data.train_set()
if lang_train_set is None:
print("No training data found for language {}.".format(
language))
else:
train_data.append(lang_train_set)
train_data = pd.concat(train_data)
# train the CWI model
cwi_model = CrosslingualCWI(list(datasets_per_language.keys()))
cwi_model.train(train_data)
# test the model
test_datasets = datasets_per_language[test_language]
for dataset_name in test_datasets:
data = Dataset(test_language, dataset_name)
print("\nTesting on {} - {}.".format(test_language, dataset_name))
if evaluation_split in ["dev", "both"]:
print("\nResults on Development Data")
predictions_dev = cwi_model.predict(data.dev_set())
gold_labels_dev = data.dev_set()['gold_label']
print(
report_binary_score(gold_labels_dev, predictions_dev,
detailed_report))
if evaluation_split in ["test", "both"]:
print("\nResults on Test Data")
predictions_test = cwi_model.predict(data.test_set())
gold_labels_test = data.test_set()['gold_label']
print(
report_binary_score(gold_labels_test, predictions_test,
detailed_report))
print()
def run_model(model_type,
domain,
is_verbose=True,
probs=False,
get_history=False):
if model_type == 'logistic_regression':
model = LR(domain, probs=probs)
elif model_type == 'fast_text':
model = FastText(domain, probs=probs, is_verbose=is_verbose)
elif model_type == 'mlp':
model = MLP(domain, is_verbose=is_verbose)
else:
raise ValueError('incorrect model type choice')
print("Loading Data")
data = Dataset()
print("Training Model")
hist = model.train(data.train_set())
if get_history is True:
return hist
else:
y, y_pred = model.test(data.test_set())
return y, y_pred
def run_model(in_notebook=False):
print("\nPredicting Speaker Stance - Baseline Model ")
print("Loading Data")
if in_notebook is True:
data = Dataset(in_notebook=True)
else:
data = Dataset()
model = Model()
print("Training Model")
model.train(data.train_set())
print("\nResults on Test Data")
y, y_pred = model.test(data.test_set())
print(report_scores(y, y_pred))
def run_model(language, dataset_name, evaluation_split, detailed_report):
"""Trains and tests the CWI model for a particular dataset of a particular language. Reports results.
Args:
language: The language of the dataset.
dataset_name: The name of the dataset (all files should have it).
evaluation_split: The split of the data to use for evaluating the performance of the model (dev or test).
detailed_report: Whether to display a detailed report or just overall score.
"""
print("\nModel for {} - {}.".format(language, dataset_name))
data = Dataset(language, dataset_name)
#The code below is used for creating unigram probability csv files
# if (language == 'spanish'):
# corpus_words = nltk.corpus.cess_esp.words()
# unigram_counts = Counter(corpus_words)
# total_words = len(corpus_words)
# def calc_unigram_prob(unigram_counts, total_words):
# u_prob = {} #defaultdict
# for word in unigram_counts:
# u_prob[word] = unigram_counts[word]/total_words
# return u_prob
# def save_to_file(u_prob,file_name):
# w = csv.writer(open(file_name, "w"))
# for word, prob in u_prob.items():
# w.writerow([word, prob])
# print('calc unigram prob: ')
# u_prob = calc_unigram_prob(unigram_counts, total_words)
# print('saving file')
# save_to_file(u_prob, 'data/external/spanish_u_prob.csv')
# kdfjei
baseline = MonolingualCWI(language)
baseline.train(data.train_set())
if evaluation_split in ["dev", "both"]:
print("\nResults on Development Data")
predictions_dev = baseline.predict(data.dev_set())
gold_labels_dev = data.dev_set()['gold_label']
print(
report_binary_score(gold_labels_dev, predictions_dev,
detailed_report))
if evaluation_split in ["test", "both"]:
print("\nResults on Test Data")
predictions_test = baseline.predict(data.test_set())
gold_labels_test = data.test_set()['gold_label']
print(
report_binary_score(gold_labels_test, predictions_test,
detailed_report))
print()
def run_model():
print("\nPredicting Speaker Stance - Baseline Model ")
print("Loading Data")
data = Dataset()
model = Model()
print("Training Model")
model.train(data.train_set())
print("\nResults on Test Data")
y, y_pred = model.test(data.test_set())
print(report_scores(y, y_pred))
def __call__(self, parameters=None, verbose=True):
Logger.log(verbose, "Loading datasets...")
Logger.indent()
p = Parameters(parameters, self.defaultParameters)
dataset = {}
for nameset in ["train", "test"]:
data, labels = gen_reg_data(p.n, p.m)
Logger.log(verbose, "synthetic " + nameset + " data generated")
dataset[nameset] = Dataset(p, data, verbose=verbose)
Logger.dindent()
Logger.log(verbose, "datasets loaded!\n")
return [dataset]
def evaluate(self, dataset, privater, epoch=-1):
print(f'evaluating {type(self).__name__} on epoch {epoch}')
train_data = dataset.get_train()
test_data = dataset.get_test()
train_data = privater.predict(train_data)
test_data = privater.predict(test_data)
dataset = Dataset(train_data=train_data, test_data=test_data)
self.train_model = self.build_model(train_data)
early_stopping = EarlyStopping()
callbacks = [early_stopping]
trainer = KerasTrainer(batch_size=self.batch_size,
epochs=self.epochs,
verbose=self.verbose)
trainer.train(dataset, self, callbacks=callbacks)
return min(early_stopping.best_val_acc, early_stopping.best_acc)
def show_recognized(predictions):
"""
Show images for which the system has not produced a valid classification.
:param images: list with image files
:param predictions: list with predictions
"""
d = Dataset(os.path.join(config.conf.AppPath, config.conf.DataPath))
images = d.test.as_dataframe()['filename']
labels = d.test.as_dataframe()['label']
if len(images) != len(predictions):
raise ValueError(
'images must contains same number of examples than predictions')
to_show = images[predictions >= 0]
for img in to_show:
plt.imshow(mpimg.imread(img))
plt.show()
def show_test_errors(predictions):
"""
Show images for which predictions is different from ground truth label.
:param images: list with image files
:param predictions: list with predictions
:param labels: list with ground truth labels
"""
d = Dataset(os.path.join(config.conf.AppPath, config.conf.DataPath))
images = d.test.as_dataframe()['filename']
labels = d.test.as_dataframe()['label']
if len(images) != len(predictions) != len(labels):
raise ValueError(
'images must contains same number of examples than predictions and labels'
)
to_show = images[(predictions >= 0) & (predictions != labels)]
for img in to_show:
plt.imshow(mpimg.imread(img))
plt.show()
def __call__(self, parameters=None, verbose=True):
Logger.log(verbose, "Loading datasets...")
Logger.indent()
p = Parameters(parameters, self.defaultParameters)
dataset = {}
for nameset in ["train", "test"]:
data, names = load_data(nameset,
k=p.k,
mat=p.mat,
small=p.small,
nsmall=p.nsmall,
givename=True)
names = "(" + " and ".join(names) + ")"
Logger.log(verbose, nameset + " data loaded! " + names)
dataset[nameset] = Dataset(p, *data, verbose=verbose)
Logger.dindent()
Logger.log(verbose, "datasets loaded!\n")
return [dataset]
def run_model(model_type, domain, trainset, testset, is_verbose, **kwargs):
if model_type == 'fast_text':
model = FastText(domain, is_verbose=is_verbose, **kwargs)
elif model_type == 'logistic_regression':
model = LR(domain)
elif model_type == 'mlp':
model = MLP(domain, is_verbose=is_verbose, **kwargs)
else:
raise ValueError('incorrect model type choice {}'.format(model_type))
print("Loading Data")
data = Dataset()
print("Training Model")
model.train(trainset)
y, y_pred = model.test(testset)
return y, y_pred
def run_experiment(config, run_training, run_testing):
"""Runs an experiment specified by a config file and cli args.
Args:
config: NamedTuple, a config.yaml file which has been parsed into an object.
run_training: bool, whether to run the training pipeline.
run_testing: bool, whether to run the testing pipeline.
Returns:
results: list(dict), a list of dictionaries, one per model specified in
the `config`. Each dictionary contains performance metrics and
hyperparameters which are specific to that model.
"""
# Boilerplate: set seeds and create working dir.
set_seed(config.seed)
if not os.path.exists(config.working_dir):
os.MakeDirs(config.working_dir)
utils.write_config(config, os.path.join(config.working_dir, 'config.yaml'))
print('MAIN: parsing dataset')
start = time.time()
dataset = Dataset(config, config.working_dir)
print('MAIN: dataset done. took %.2fs' % (time.time() - start))
# Train & test each of the models which are listed in the config.
results = []
for model_description in config.model_spec:
if model_description.get('skip', False):
continue
# try:
if run_training:
train_model(model_description, config, dataset)
if run_testing:
results += test_model(model_description, config, dataset)
#except Exception as e:
# print(str(e))
return results
def KernelTest(kernelname, parameters, synth=False):
Dataset = findData("allseq")()[0]
if synth:
import numpy as np
from src.data.dataset import Dataset
defaultParameters = {
"k": 0,
"mat": False,
"shuffle": False,
"small": True,
"nsmall": 200,
"labels_change": True,
"name": "seq",
"nclasses": 2
}
from src.tools.utils import Parameters
p = Parameters(None, defaultParameters)
train = Dataset(p, np.array(['ATTA', 'AAAA']), np.array([0, 1]))
else:
train = Dataset["train"]
Kernel = findKernel(kernelname)
Logger.log(True, "Test the " + kernelname + " kernel.")
Logger.indent()
kernels = []
for params in parameters:
Logger.log(True, "Test with these parameters: " + str(params))
Logger.indent()
kernel = Kernel(train, params)
kernels.append(kernel)
Logger.log(True, kernel.K)
Logger.dindent()
# ipdb.set_trace()
Logger.dindent()
def run_model(language, dataset_name, evaluation_split, detailed_report, ablate):
"""Trains and tests the CWI model for a particular dataset of a particular language. Reports results.
Args:
language: The language of the dataset.
dataset_name: The name of the dataset (all files should have it).
evaluation_split: The split of the data to use for evaluating the performance of the model (dev or test).
detailed_report: Whether to display a detailed report or just overall score.
"""
score_only = True if ablate else False
data = Dataset(language, dataset_name)
#The code below is used for creating unigram probability csv files
#corp = nltk.corpus.ConllCorpusReader('.', 'tiger_release_aug07.corrected.16012013.conll09',
# ['ignore', 'words', 'ignore', 'ignore', 'ignore'],
#encoding='utf-8')
# filename = 'europarl-v7.fr-en.fr'
# file = open(filename, mode='rt', encoding='utf-8')
# corpus_words = []
# for line in file:
# #print(line)
# corpus_words += line.strip(',').strip('.').split()
# #print(corpus_words)
# #corpus_words = corp.words()
# unigram_counts = Counter(corpus_words)
# total_words = len(corpus_words)
# def calc_unigram_prob(unigram_counts, total_words):
# u_prob = {} #defaultdict
# for word in unigram_counts:
# u_prob[word] = unigram_counts[word]/total_words
# return u_prob
# def save_to_file(u_prob,file_name):
# w = csv.writer(open(file_name, "w"))
# for word, prob in u_prob.items():
# w.writerow([word, prob])
# print('calc unigram prob: ')
# u_prob = calc_unigram_prob(unigram_counts, total_words)
# print('saving file')
# save_to_file(u_prob, 'data/external/french_u_prob.csv')
# hgiuyo
baseline = MonolingualCWI(language, ablate)
baseline.train(data.train_set())
if evaluation_split in ["dev", "both"]:
if not score_only:
print("\nResults on Development Data")
predictions_dev = baseline.predict(data.dev_set())
gold_labels_dev = data.dev_set()['gold_label']
print(report_binary_score(gold_labels_dev, predictions_dev, detailed_report, score_only))
if evaluation_split in ["test", "both"]:
if not score_only:
print("\nResults on Test Data")
predictions_test = baseline.predict(data.test_set())
gold_labels_test = data.test_set()['gold_label']
print(report_binary_score(gold_labels_test, predictions_test, detailed_report, score_only))
if not score_only:
print()
def run_experiment(config, args, expt_id):
# if train, switch the dataset to train, then
# train and save each model in the config spec
if not os.path.exists(config.working_dir):
os.makedirs(config.working_dir)
utils.write_config(config, os.path.join(config.working_dir, 'config.yaml'))
print 'MAIN: parsing dataset'
d = Dataset(config, config.base_dir)
print 'MAIN: dataset done. took %.2fs' % (time.time() - start)
if args.train:
d.set_active_split(config.train_suffix)
for model_description in config.model_spec:
if model_description.get('skip', False):
continue
if args.model is not None and args.model != model_description[
'type']:
continue
print 'MAIN: training ', model_description['name']
start_time = time.time()
model_dir = os.path.join(config.working_dir,
model_description['name'])
if not os.path.exists(model_dir):
os.makedirs(model_dir)
model = constants.MODEL_CLASSES[model_description['type']](
config=config, params=model_description['params'])
model.train(d, model_dir)
model.save(model_dir)
print 'MAIN: training %s done, time %.2fs' % (
model_description['name'], time.time() - start_time)
# if test, switch thh datset to test,
# and run inference + evaluation for each model
# in the config spec
if args.test:
d.set_active_split(config.test_suffix)
results = [] # items to be written in executive summary
for model_description in config.model_spec:
if model_description.get('skip', False):
continue
if args.model is not None and args.model != model_description[
'type']:
continue
print 'MAIN: inference with ', model_description['name']
start_time = time.time()
model = constants.MODEL_CLASSES[model_description['type']](
config=config, params=model_description['params'])
model_dir = os.path.join(config.working_dir,
model_description['name'])
model.load(d, model_dir)
predictions = model.inference(d, model_dir)
utils.pickle(predictions, os.path.join(model_dir, 'predictions'))
evaluation = evaluator.evaluate(config, d, predictions, model_dir)
utils.pickle(evaluation, os.path.join(model_dir, 'evaluation'))
evaluator.write_summary(evaluation, model_dir)
# store info for executive summary
results.append({
'model-name': model_description['name'],
'model-type': model_description['type'],
'params': str(model_description['params']),
'correlation': evaluation['mu_corr'],
'regression_performance': evaluation['mu_reg_perf'],
'fixed_performance': evaluation['mu_fixed_perf'],
'model_dir': model_dir,
'expt_id': expt_id
})
print 'MAIN: evaluation %s done, time %.2fs' % (
model_description['name'], time.time() - start_time)
return results
# coding: utf-8 from src.models.baseline import Model from src.data.dataset import Dataset data = Dataset().train_set() y = Model().mlb.fit_transform(data)
y = self.target_pipe.fit_transform(trainset)
num_features = X.shape[1]
self.model = self.build_net(num_features)
self.model.fit(X, y, epochs=500, batch_size=32, verbose=self.is_verbose)
def test(self, testset):
X = self.feature_pipe.transform(testset)
y = self.mlb.transform(testset)
print(y.shape)
y_pred_raw = self.model.predict(X)
y_pred = np.column_stack(y_pred_raw) # join the raw outputs into format for sklearn scoring
print(y_pred.shape)
threshold = 0.5
for row in y_pred:
for i, val in enumerate(row):
if val > threshold:
row[i] = 1
elif val < threshold:
row[i] = 0
print(y_pred[0])
return y, y_pred
if __name__ == '__main__':
from src.data.dataset import Dataset
data = Dataset()
model = Model()
model.train(data.train_set())
model.test(data.test_set())
def dataset():
''' Return Dataset that has loaded the vocab file '''
vocab = Vocab(VOCAB_PATH)
return Dataset(vocab)
'20201005-183234-ig_resnext101_32x8d-320',
'20201012-184118-ig_seresnext101_32x8d-224',
'20201014-232454-ig_resnext101_32x16d-320'
]
for checkpoint in checkpoint_list:
name = '-'.join(checkpoint.split('-')[1:])
model_list.append(name)
for index, model_name in enumerate(model_list):
img_size = int(model_name.split('-')[-1])
if args.flag == "valid":
save_weights = True
else:
save_weights = False
dataset = Dataset(os.path.join(data_root, args.flag))
checkpoint = glob.glob(
os.path.join(output_root,
checkpoint_list[index] + '/*best*.pth.tar'))[0]
model = create_model('%s' % model_name.split('-')[-2],
num_classes=args.num_classes,
in_chans=3,
checkpoint_path='%s' % checkpoint)
model = model.cuda()
model.eval()
config = resolve_data_config(vars(args), model=model)
loader = create_loader(dataset,
input_size=img_size,
batch_size=args.batch_size,
'20200923-111323-gluon_seresnext101_32x4d-224',
'20200928-091837-hrnet_w44-224'
]
for checkpoint in checkpoint_list:
name = '-'.join(checkpoint.split('-')[1:])
model_list.append(name)
for index, model_name in enumerate(checkpoint_list):
img_size = int(model_name.split('-')[-1])
if args.flag == "valid":
data_path = os.path.join(
"/home/data/classification/action/new_data/valid")
else:
data_path = "/home/data/classification/action/new_data/test/"
if data_path.split('/')[-1] == 'valid':
dataset = Dataset(data_path,
transform=tta_test_transform(img_size))
save_weights = True
else:
dataset = TestDataset(data_path,
transform=tta_test_transform(img_size))
save_weights = False
checkpoint = glob.glob(
os.path.join(output_root,
checkpoint_list[index] + '/*best*.pth.tar'))[0]
model = create_model('%s' % model_name.split('-')[-2],
num_classes=args.num_classes,
in_chans=3,
checkpoint_path='%s' % checkpoint)
model = model.cuda()
model.eval()
counter.update(tokens)
sort_by_counts = sorted(counter.items(), key=lambda x: x[1])
words, counts = zip(*sort_by_counts)
word2idx = dict(zip(words, range(1, len(words) + 1)))
return word2idx, max_len
def fit(self, X, *_):
self.word2idx, self.sent_size = self.build_vocab(X)
return self
def transform(self, X, *_):
vec = np.zeros((len(X), self.sent_size + 25))
for i, sent in enumerate(X):
tokens = self.TK.tokenize(sent, lowercase=True)
for j, tok in enumerate(tokens):
vec[i][j] = self.word2idx[tok]
return vec
if __name__ == '__main__':
from src.data.dataset import Dataset
trainset = Dataset().train_set()
WI = WordIndexer()
test = WI.fit_transform(trainset['Utterance'])
print(test[0])
print(len(WI.word2idx))
print(WI.sent_size)
def main(params):
# initialize the experiment
logger = initialize_exp(params)
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" % ", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in ['n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index', 'mask_index']:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'], reloaded['dico_counts'])
encoder = TransformerModel(model_params, dico, is_encoder=True, with_output=True).cuda().eval()
decoder = TransformerModel(model_params, dico, is_encoder=False, with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder.load_state_dict(reloaded['decoder'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
# float16
if params.fp16:
assert torch.backends.cudnn.enabled
encoder = network_to_half(encoder)
decoder = network_to_half(decoder)
input_data = torch.load(params.input)
eval_dataset = Dataset(input_data["sentences"], input_data["positions"], params)
if params.subset_start is not None:
assert params.subset_end
eval_dataset.select_data(params.subset_start, params.subset_end)
eval_dataset.remove_empty_sentences()
eval_dataset.remove_long_sentences(params.max_len)
n_batch = 0
out = io.open(params.output_path, "w", encoding="utf-8")
inp_dump = io.open(os.path.join(params.dump_path, "input.txt"), "w", encoding="utf-8")
logger.info("logging to {}".format(os.path.join(params.dump_path, 'input.txt')))
with open(params.output_path, "w", encoding="utf-8") as out:
for batch in eval_dataset.get_iterator(shuffle=False):
n_batch += 1
(x1, len1) = batch
input_text = convert_to_text(x1, len1, input_data["dico"], params)
inp_dump.write("\n".join(input_text))
inp_dump.write("\n")
langs1 = x1.clone().fill_(params.src_id)
# cuda
x1, len1, langs1 = to_cuda(x1, len1, langs1)
# encode source sentence
enc1 = encoder("fwd", x=x1, lengths=len1, langs=langs1, causal=False)
enc1 = enc1.transpose(0, 1)
# generate translation - translate / convert to text
max_len = int(1.5 * len1.max().item() + 10)
if params.beam_size == 1:
generated, lengths = decoder.generate(enc1, len1, params.tgt_id, max_len=max_len)
else:
generated, lengths = decoder.generate_beam(
enc1, len1, params.tgt_id, beam_size=params.beam_size,
length_penalty=params.length_penalty,
early_stopping=params.early_stopping,
max_len=max_len)
hypotheses_batch = convert_to_text(generated, lengths, input_data["dico"], params)
out.write("\n".join(hypotheses_batch))
out.write("\n")
if n_batch % 100 == 0:
logger.info("{} batches processed".format(n_batch))
out.close()
inp_dump.close()
def main(unused_argv):
if len(unused_argv) != 1:
raise Exception('Problem with number of flags entered %s' %
unused_argv)
tf.logging.set_verbosity(tf.logging.INFO)
tf.logging.info('Starting RVAE model in %s mode', (FLAGS.mode))
if FLAGS.mode == 'train':
assert FLAGS.eval_path is not None, "Error! Eval path must be provided in 'train' mode. Use train_only for only training"
elif FLAGS.mode == 'predict':
FLAGS.batch_size = 1
# change model_dir to model_dir/exp_name and create dir if needed
FLAGS.model_dir = os.path.join(FLAGS.model_dir, FLAGS.exp_name)
if not os.path.exists(FLAGS.model_dir):
if FLAGS.mode == 'train' or FLAGS.mode == 'save_embed':
os.makedirs(FLAGS.model_dir)
else:
raise Exception(
"The model_dir specified does not exist. Run in train to create it"
)
elif not os.path.exists(FLAGS.vocab_path):
raise Exception("Path specified for vocab file does not exist")
elif FLAGS.checkpoint_path and not os.path.exists(FLAGS.checkpoint_path):
raise Exception("Path for checkpoint does not exist")
# load vocab and calculate size
vocab = Vocab(vocab_path=FLAGS.vocab_path)
vsize = len(vocab.vocab)
# load the dataset
ds = Dataset(vocab)
# create an hps list
if not FLAGS.app_config or not FLAGS.model_params:
hp_list = [
'batch_size', 'emb_dim', 'hidden_dim', 'latent_dim', 'dec_layers',
'beam_size', 'max_dec_steps', 'lr', 'keep_prob', 'use_wdrop',
'model_dir', 'vocab_path'
]
hps_dict = {}
for key in FLAGS:
if key in hp_list:
hps_dict[key] = FLAGS[key].value
hps = namedtuple("HParams", hps_dict.keys())(**hps_dict)
else:
# TODO: Fix the yaml file location to use relative pathing so that we can find the config files
flags = AppConfig('app.yaml', 'default')
hps = ModelParams('hps.yaml', 'default')
# call the model
model = RVAE(hps, vsize)
if FLAGS.mode == 'train':
train_and_eval(model, ds, vocab)
elif FLAGS.mode == 'predict':
predictions = infer(model, ds, vocab, FLAGS.checkpoint_path)
print(predictions)
elif FLAGS.mode == 'eval':
eval(model, ds, vocab)
elif FLAGS.mode == 'save_embed':
_ = vocab.read_embeddings(path=FLAGS.embed_path, load_np=False)
print("Done saving numpy matrix")
return
elif FLAGS.mode == 'debug':
print("debug")
else:
raise Exception("Invalid mode argument")
return
def run_model(selective_testing, translate, test_language, evaluation_split,
detailed_report):
""" Trains the CWI model in all languages but one. Tests on all datasets of
a particular language. Reports results.
Args:
test_language: The language of the dataset to use for testing.
evaluation_split: The split of the data to use for
evaluating the performance of the model (dev or
test).
detailed_report: Whether to display a detailed report or just overall
score.
"""
# collect the training data for all the languages but one
train_data = []
if selective_testing == 'ESG':
for language, datasets_names in datasets_per_language.items():
if language != test_language:
for dataset_name in datasets_names:
data = Dataset(language, dataset_name)
lang_train_set = data.train_set()
if lang_train_set is None:
print("No training data found for language {}.".format(
language))
else:
train_data.append(lang_train_set)
train_data = pd.concat(train_data)
else:
train_data = pd.DataFrame()
if 'E' in selective_testing:
train_data = pd.concat([
train_data,
Dataset('english', 'News').train_set(),
Dataset('english', 'WikiNews').train_set(),
Dataset('english', 'Wikipedia').train_set()
])
if 'S' in selective_testing:
train_data = pd.concat(
[train_data,
Dataset('spanish', 'Spanish').train_set()])
if 'G' in selective_testing:
train_data = pd.concat(
[train_data,
Dataset('german', 'German').train_set()])
# if selective_testing == 'ES':
# train_data = pd.concat([Dataset('english','News').train_set(),
# Dataset('english','WikiNews').train_set(),
# Dataset('english','Wikipedia').train_set(),
# Dataset('spanish','Spanish').train_set()])
# elif selective_testing == 'EG':
# train_data = pd.concat([Dataset('english','News').train_set(),
# Dataset('english','WikiNews').train_set()
# ,Dataset('english','Wikipedia').train_set(),
# Dataset('german','German').train_set()])
# elif selective_testing == 'E':
# train_data = pd.concat([Dataset('english','News').train_set(),
# Dataset('english','WikiNews').train_set()
# ,Dataset('english','Wikipedia').train_set()])
# elif selective_testing == 'G':
# train_data = pd.concat([Dataset('german','German').train_set()])
#
# elif selective_testing == 'S':
# train_data = pd.concat([Dataset('spanish','Spanish').train_set()])
# else:
# train_data = pd.concat([Dataset('spanish','Spanish').train_set(),
# Dataset('german','German').train_set()])
# train the CWI model
cwi_model = CrosslingualCWI(list(datasets_per_language.keys()))
cwi_model.train(train_data)
# test the model
test_datasets = datasets_per_language[test_language]
for dataset_name in test_datasets:
data = Dataset(test_language, dataset_name)
print("\nTesting on {} - {}.".format(test_language, dataset_name))
if evaluation_split in ["dev", "both"]:
print("\nResults on Development Data")
if test_language == 'french':
print("\nNo Dev Data for French, skipping...")
else:
predictions_dev = cwi_model.predict(data.dev_set())
gold_labels_dev = data.dev_set()['gold_label']
print(
report_binary_score(gold_labels_dev, predictions_dev,
detailed_report))
if evaluation_split in ["test", "both"]:
print("\nResults on Test Data")
data.translate = translate
predictions_test = cwi_model.predict(data.test_set())
gold_labels_test = data.test_set()['gold_label']
print(
report_binary_score(gold_labels_test, predictions_test,
detailed_report))
print()