def __init__(self,
task,
dataset,
label_helper,
user,
model_config,
config,
model_labelling=True):
self.config = config.config
self.task = task
self.dataset = dataset
self.data_type = self.dataset.data_type
self.label_helper = label_helper
model_directory = model_config['directory']
self.model = model_builder.ModelBuilder(dataset, self.label_helper,
model_config).build()
self.trainer = Trainer(model_directory,
self.model,
self.dataset,
self.label_helper,
logger=logger)
self.trainer.load_existing()
self.labeller = ModelLabeller(self.model,
self.dataset,
self.label_helper,
logger=logger)
self.user = user
self.model_labelling = model_labelling
class TestOutbrainTrainer(unittest.TestCase):
def setUp(self):
self.q_file = '../data/query_groups.txt'
self.i_file = '../data/input_groups_2.txt'
self.tr = Trainer()
def test_read_file(self):
data = self.tr.read_file(self.q_file)
self.assertEqual(type(data), list)
for line in data:
self.assertEqual(type(line), list)
for word in line:
self.assertEqual(type(word), str)
def test_output(self):
data = {"blue": 1, "red": 2, "yellow": 1}
result = self.tr.output(data)
self.assertEqual(type(result), str)
def test_get_list(self):
results = self.tr.get_list(self.q_file)
for r in results:
self.assertEqual(type(r), set)
def test_match_query_input(self):
results = self.tr.match_query_input(self.i_file, self.q_file)
self.assertEqual(type(results), str)
def train(config: Config, cyclegan: CycleGAN):
"""
Train CycleGAN using the hyperparameters in config.
"""
# Initialize trainer
trainer = Trainer(config, cyclegan, dataset)
# Callback for saving checkpoints.
def checkpoint(epoch):
if epoch % config.checkpoint_interval == 0:
save_checkpoint(epoch, cyclegan, config.n_epochs, dir_checkpoints)
# Callback for saving snapshots.
def snapshot(epoch, batch):
if batch % config.snapshot_interval == 0:
save_snapshot(epoch, batch, cyclegan, dataset.batch_test(1),
config.image_shape, dir_snapshots)
# Register callbacks
trainer.on_epoch_start.append(checkpoint)
trainer.on_iteration_start.append(snapshot)
# Train the model
trainer.train(n_epochs=config.n_epochs,
batch_size=config.batch_size,
start_epoch=epoch)
def main():
config = json.load(open('./config.json'))
data_manager = DataManager(config)
model = FConvNet()
if config['use_cuda'] is True:
model = model.cuda()
model.apply(FConvNet.init_weights)
criterion = torch.nn.MSELoss()
optimizer = optim.Adam(model.parameters(),
lr=config['lr'],
weight_decay=config['weight_decay'])
train_loader, validation_loader = data_manager.get_train_eval_dataloaders(
config['train_data_path'])
trainer = Trainer(model, train_loader, validation_loader, criterion,
optimizer, config)
trainer.train()
def set_trainer(config):
# load a checkpoint
if config.checkpoint is not None:
# load data
train_loader = load_data(config, 'train', False)
model, optimizer, word_map, start_epoch = load_checkpoint(config.checkpoint, device)
print('\nLoaded checkpoint from epoch %d.\n' % (start_epoch - 1))
# or initialize model
else:
start_epoch = 0
# load data
train_loader, embeddings, emb_size, word_map, n_classes, vocab_size = load_data(config, 'train', True)
model = models.setup(
config = config,
n_classes = n_classes,
vocab_size = vocab_size,
embeddings = embeddings,
emb_size = emb_size
)
optimizer = optim.Adam(
params = filter(lambda p: p.requires_grad, model.parameters()),
lr = config.lr
)
# loss functions
loss_function = nn.CrossEntropyLoss()
# move to device
model = model.to(device)
loss_function = loss_function.to(device)
trainer = Trainer(
num_epochs = config.num_epochs,
start_epoch = start_epoch,
train_loader = train_loader,
model = model,
model_name = config.model_name,
loss_function = loss_function,
optimizer = optimizer,
lr_decay = config.lr_decay,
word_map = word_map,
grad_clip = config.grad_clip,
print_freq = config.print_freq,
checkpoint_path = config.checkpoint_path,
checkpoint_basename = config.checkpoint_basename
)
return trainer
def initialize_model_and_trainer(model_properties, training_properties,
datasetloader, device):
logger.info("Model type is %s", training_properties["learner"])
if training_properties["learner"] == "text_cnn":
model = TextCnn(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "gru":
model = GRU(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "lstm":
model = LSTM(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "char_cnn":
model = CharCNN(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "vdcnn":
model = VDCNN(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "conv_deconv_cnn":
model = ConvDeconvCNN(model_properties)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "transformer_google":
model = TransformerGoogle(model_properties).model.to(device)
trainer = Trainer.trainer_factory("single_model_trainer",
training_properties, datasetloader,
device)
elif training_properties["learner"] == "lstmcrf":
assert training_properties["task"] == "ner"
model = LSTMCRF(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_ner_trainer",
training_properties, datasetloader,
device)
else:
raise ValueError(
"Model is not defined! Available learner values are : 'text_cnn', 'char_cnn', 'vdcnn', 'gru', "
"'lstm', 'conv_deconv_cnn' and 'transformer_google'")
return model, trainer
def main():
# Some boilerplate code for easier logging.
logger.remove(0)
logger.add(sys.stdout, level="INFO")
logger.add("../logs/{time}.log", level="DEBUG")
start_time = datetime.now()
logger.info("Started at: {}".format(start_time))
# Parsing main arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--cfg_path',
required=True,
help='Path to the configuration.ini file to work with.')
parser.add_argument('--train',
action='store_true',
default=False,
help='Training the model')
# parser.add_argument('--validation', action='store_true', default=False, help='Training the model')
# parser.add_argument('--test', action='store_true', default=False, help='Testing the model')
args = parser.parse_args()
# Read Configuration file
logger.info("Loading config file from path: {}".format(args.cfg_path))
config = AttnGANConfig(args.cfg_path)
config.log_config_info()
# Pytorch stuff:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if args.train:
dataset_name = config.dataset['name']
logger.info("Starting training on {} dataset".format(dataset_name))
# Training
trainer = Trainer(config)
trainer.train()
def get_trainer(model, dataset, name, frequency):
trainer = Trainer(model)
trainer.on_iteration_end.append(
checkpoint(model, name, frequency=frequency))
trainer.on_iteration_end.append(
log(model, dataset, name=name, frequency=frequency))
lrs = lr_scheduler.ReduceLROnPlateau(model.optimizer,
factor=0.1,
patience=5,
verbose=True)
trainer.on_iteration_end.append(
LearningRateScheduleWrapper(lrs, frequency=frequency))
return trainer
def train_from_config(config, serialization_dir):
"""
Trains model from given config.
:param config: config file
:param serialization_dir: serialization directory path
"""
rng = prepare_environment(config)
experiment = Experiment.from_config(config, rng=rng)
experiment.print_configs()
trainer = Trainer.from_params(config=experiment.config['trainer'],
model=experiment.model,
train_stream=experiment.streams.train,
dev_stream=experiment.streams.dev,
serialization_dir=serialization_dir)
trainer.train()
def initialize_model_and_trainer(model_properties, training_properties, datasetloader, device):
logger.info("Model type is %s", training_properties["learner"])
if training_properties["learner"] == "text_cnn":
model = TextCnn(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "gru":
model = GRU(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "lstm":
model = LSTM(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "char_cnn":
model = CharCNN(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "vdcnn":
model = VDCNN(model_properties).to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "conv_deconv_cnn":
convDeconveCNN = ConvDeconvCNN(model_properties)
encoderCNN = convDeconveCNN.encoder.to(device)
decoderCNN = convDeconveCNN.decoder.to(device)
classifier = convDeconveCNN.classifier.to(device)
trainer = Trainer.trainer_factory("multiple_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
model = [encoderCNN, decoderCNN, classifier]
elif training_properties["learner"] == "transformer_google":
model = TransformerGoogle(model_properties).model.to(device)
trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
datasetloader.val_iter, datasetloader.test_iter, device)
elif training_properties["learner"] == "crf":
model = ConditionalRandomField().to(device)
else:
raise ValueError("Model is not defined! Available learner values are : 'text_cnn', 'char_cnn', 'vdcnn', 'gru', "
"'lstm', 'conv_deconv_cnn' and 'transformer_google'")
return model, trainer
learning_params['learning_rate'] = choice([1.e-1, 1.e-2, 1.e-3, 1.e-4])
learning_params['weight_decay'] = choice([1, 1.e-2, 1.e-4, 0])
learning_params['minibatch_size'] = choice([16, 32, 64, 128])
test_dataset, training_dataset, validation_dataset = extractor.generate_datasets(
device=device,
test_subject_id=test_subject_id,
validation_subject_id=validation_subject_id,
window_secs=window_secs,
downsampling_factor=downsampling_factor,
window_shift=window_shift)
trainer = Trainer(learning_params=learning_params,
data_params=data_params,
device=device,
output_path=output_folder,
test_dataset=test_dataset,
training_dataset=training_dataset,
validation_dataset=validation_dataset)
output_len = 12
if args.model_type == 'cnn':
input_len = next(iter(trainer.train_dataloader))[0].shape[1]
num_input_channels = next(iter(trainer.train_dataloader))[0].shape[2]
encoder_params = CNN.generate_params()
decoder_params = MLP.generate_params()
model = CNN(
num_input_channels=num_input_channels,
input_size=input_len,
output_len=output_len,
encoder_params=encoder_params,
coco_root = '/data/COCO/'
backbone = 'resnet101' # 'resnet50'
opt = 'adam'
inp_size = 480 # input size 480*480
feat_stride = 4
node_count = 1024 # Hidden Layer Node Count
coeff = 2 # Coefficient of bbox size
threshold = 0.21 # BBOX threshold
num_of_keypoints = 3 # Minimum number of keypoints for each bbox in training
# model parameters in MultiPoseNet
prn_para = ['prn']
#####################################################################
# Set Training parameters
params = Trainer.TrainParams()
params.exp_name = 'prn_subnet/'
params.subnet_name = 'prn_subnet'
params.save_dir = './extra/models/{}'.format(params.exp_name)
params.ckpt = './demo/models/ckpt_baseline_resnet101.h5'
params.max_epoch = 40
params.init_lr = 1.0e-3
params.lr_decay = 0.9
params.gpus = [0]
params.batch_size = 8 * len(params.gpus)
params.val_nbatch_end_epoch = 2000
params.print_freq = 1000
def set_trainer():
# data parameters
data_folder = config.dataset_output_path
data_name = config.dataset_basename
# GPU / CPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# load word2id map
word_map_file = os.path.join(data_folder, 'wordmap_' + data_name + '.json')
with open(word_map_file, 'r') as j:
word_map = json.load(j)
# create id2word map
rev_word_map = {v: k for k, v in word_map.items()}
# initialize encoder-decoder framework
if config.checkpoint is None:
start_epoch = 0
epochs_since_improvement = 0
best_bleu4 = 0.
caption_model = config.caption_model
# ------------- word embeddings -------------
if config.embed_pretrain == True:
# load pre-trained word embeddings for words in the word map
embeddings, embed_dim = load_embeddings(
emb_file=config.embed_path,
word_map=word_map,
output_folder=config.dataset_output_path,
output_basename=config.dataset_basename)
else:
# or initialize embedding weights randomly
embeddings = None
embed_dim = config.embed_dim
# ----------------- encoder ------------------
encoder = Encoder.setup(embed_dim=embed_dim)
encoder.CNN.fine_tune(config.fine_tune_encoder)
# optimizer for encoder's CNN (if fine-tune)
if config.fine_tune_encoder:
encoder_optimizer = torch.optim.Adam(params=filter(
lambda p: p.requires_grad, encoder.CNN.parameters()),
lr=config.encoder_lr)
else:
encoder_optimizer = None
# ----------------- decoder ------------------
decoder = Decoder.setup(vocab_size=len(word_map),
embed_dim=embed_dim,
embeddings=embeddings)
# optimizer for decoder
# print(len(list(decoder.parameters())))
decoder_params = list(
filter(lambda p: p.requires_grad, decoder.parameters()))
# print(len(decoder_params))
if caption_model == 'adaptive_att' or caption_model == 'spatial_att':
decoder_params = decoder_params + list(encoder.global_mapping.parameters()) \
+ list(encoder.spatial_mapping.parameters())
elif caption_model == 'show_tell':
decoder_params = decoder_params + list(
encoder.output_layer.parameters())
decoder_optimizer = torch.optim.Adam(params=decoder_params,
lr=config.decoder_lr)
# or load checkpoint
else:
encoder,
encoder_optimizer,
decoder,
decoder_optimizer,
start_epoch,
epochs_since_improvement,
best_bleu4
caption_model = load_checkpoint(config.checkpoint,
config.fine_tune_encoder,
config.encoder_lr)
# move to GPU, if available
decoder = decoder.to(device)
encoder = encoder.to(device)
# loss function (cross entropy)
loss_function = nn.CrossEntropyLoss().to(device)
# custom dataloaders
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(CaptionDataset(
data_folder,
data_name,
'train',
transform=transforms.Compose([normalize])),
batch_size=config.batch_size,
shuffle=True,
num_workers=config.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(CaptionDataset(
data_folder,
data_name,
'val',
transform=transforms.Compose([normalize])),
batch_size=config.batch_size,
shuffle=True,
num_workers=config.workers,
pin_memory=True)
trainer = Trainer(caption_model=caption_model,
epochs=config.epochs,
device=device,
word_map=word_map,
rev_word_map=rev_word_map,
start_epoch=start_epoch,
epochs_since_improvement=epochs_since_improvement,
best_bleu4=best_bleu4,
train_loader=train_loader,
val_loader=val_loader,
encoder=encoder,
decoder=decoder,
encoder_optimizer=encoder_optimizer,
decoder_optimizer=decoder_optimizer,
loss_function=loss_function,
grad_clip=config.grad_clip,
tau=config.tau,
fine_tune_encoder=config.fine_tune_encoder,
tensorboard=config.tensorboard,
log_dir=config.log_dir)
return trainer
class LabelApp:
@staticmethod
def load_from(config_meta):
with open(config_meta['path']) as f:
config = yaml.load(f)
parser = ConfigParser(config)
parser._create_directories()
task = Task.load_from(parser.task)
dataset = Dataset.load_from(parser.dataset)
model_config = config['model']
label_helper = Label.load_from(parser.label)
user = config['user']
# Set up logger
log_level = config_meta['log_level']
logger = logging.getLogger('label_app')
logger.setLevel(getattr(logging, log_level))
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(ch)
return LabelApp(task, dataset, label_helper, user, model_config,
parser, logger)
def __init__(self,
task,
dataset,
label_helper,
user,
model_config,
config,
logger,
model_labelling=True):
self.config = config.config
self.task = task
self.dataset = dataset
self.data_type = self.dataset.data_type
self.label_helper = label_helper
model_directory = model_config['directory']
self.model = model_builder.ModelBuilder(dataset, self.label_helper,
model_config).build()
self.trainer = Trainer(model_directory,
self.model,
self.dataset,
self.label_helper,
logger=logger)
self.trainer.load_existing()
self.labeller = ModelLabeller(self.model,
self.dataset,
self.label_helper,
logger=logger)
self.user = user
self.model_labelling = model_labelling
self.logger = logger
def score(self, x):
scores = self.model.score(x)
return scores
def predict(self, x):
predictions = self.model.predict(x)
return predictions
@property
def is_done(self):
return len(self.dataset.unlabelled) == 0
def next_model_labelled_batch(self, size=100):
model_labelled, target_class = self.dataset.model_labelled(size)
return model_labelled, target_class
def next_batch(self,
size=10,
force_stage=None,
reverse_entropy=False,
prediction=False):
if self.is_done:
raise ValueError(
"Tried to sample a batch when there is nothing else to sample")
self.logger.debug("Sampling a batch for {} set.".format(
self.dataset.current_stage))
self.dataset.set_current_stage()
current_stage = force_stage if force_stage else self.dataset.current_stage
if current_stage == Dataset.TEST:
sampled_df = self.dataset.sample(size)
return sampled_df, current_stage, [], [0.5] * len(
sampled_df) # TODO: This needs to be fixed
# Generate training data
sampled_df = self.dataset.sample(size * 5)
if self.data_type == Dataset.IMAGE_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
if self.data_type == Dataset.TEXT_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
if self.data_type == Dataset.JSON_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
scores = self.model.score(x_data)
entropy_func = lambda scores: np.sum(scores * np.log(1 / scores),
axis=-1)
if len(scores.shape) == 3:
entropy = np.array(
[entropy_func(score).mean() for score in scores])
else:
entropy = entropy_func(scores)
assert len(entropy.shape) == 1
num = min(size, len(entropy) - 1)
if reverse_entropy:
entropy_indexes = np.argpartition(entropy, num)[:num]
else:
entropy_indexes = np.argpartition(-entropy, num)[:num]
# Make predictions
# TODO: This doesn't work for text or json types
if self.data_type == Dataset.IMAGE_TYPE:
x_to_score = x_data[entropy_indexes]
else:
x_to_score = []
y_prediction = None
if prediction and len(x_to_score) > 0:
y_prediction = self.predict(x_to_score)
response = (
sampled_df.iloc[entropy_indexes],
current_stage,
y_prediction,
entropy[entropy_indexes].tolist(),
)
return response
def search(self, search_query: str, num_results: int = 20):
results = self.dataset.search(search_query, num_results)
return results
def labelled_data(self, start_idx, end_idx, labelled=None):
df = self.dataset.dataset
if labelled is not None:
df = df[df['labelled'] == labelled]
if start_idx >= len(df):
return [], True
rows = df.iloc[start_idx:end_idx]
return rows, False
def add_labels(self, labels, avg_time_taken):
for label in labels:
self.dataset.add_label(
label['path'],
label['label'],
Dataset.TRAIN,
user=self.user,
time_taken=avg_time_taken,
)
def add_label(self, _id, label, time_taken):
# Validate label
# TODO: Reevaluate this get_data thing, I'm not a fan of this.
data = self.dataset.get_data(_id)
self.label_helper.validate(data, label)
label = self.label_helper.decode(label)
# _id is just the path to the file
self.dataset.add_label(
_id,
label,
self.dataset.current_stage,
user=self.user,
save=True,
time_taken=time_taken,
)
@property
def title(self):
return self.task.title
@property
def description(self):
return self.task.description
@property
def template(self):
return self.task.template
def threaded_train(self):
self.trainer.train()
def threaded_label(self):
self.labeller.start()
def get_history(self):
return self.trainer.get_history()
def get_stats(self):
return self.dataset.stats
class LabelApp:
@staticmethod
def load_from(config_path):
with open(config_path) as f:
config = yaml.load(f)
parser = ConfigParser(config)
parser._create_directories()
task = Task.load_from(parser.task)
dataset = Dataset.load_from(parser.dataset)
model_config = config['model']
label_helper = Label.load_from(parser.label)
user = config['user']
return LabelApp(task, dataset, label_helper, user, model_config,
parser)
def __init__(self,
task,
dataset,
label_helper,
user,
model_config,
config,
model_labelling=True):
self.config = config.config
self.task = task
self.dataset = dataset
self.data_type = self.dataset.data_type
self.label_helper = label_helper
model_directory = model_config['directory']
self.model = model_builder.ModelBuilder(dataset, self.label_helper,
model_config).build()
self.trainer = Trainer(model_directory,
self.model,
self.dataset,
self.label_helper,
logger=logger)
self.trainer.load_existing()
self.labeller = ModelLabeller(self.model, self.dataset, logger=logger)
self.user = user
self.model_labelling = model_labelling
def score(self, x):
scores = self.model.score(x)
return scores
def predict(self, x):
predictions = self.model.predict(x)
return predictions
@property
def is_done(self):
return len(self.dataset.unlabelled) == 0
def next_batch(self, size=10, force_stage=None, reverse_entropy=False):
if self.is_done:
raise ValueError(
"Tried to sample a batch when there is nothing else to sample")
logger.debug("Sampling a batch for {} set.".format(
self.dataset.current_stage))
self.dataset.set_current_stage()
current_stage = force_stage if force_stage else self.dataset.current_stage
if current_stage == Dataset.TEST:
sampled_df = self.dataset.sample(size)
return sampled_df, current_stage, [], [0.5] * len(
sampled_df) # TODO: This needs to be fixed
# Generate training data
sampled_df = self.dataset.sample(size * 5)
if self.data_type == Dataset.IMAGE_TYPE:
x_data = utils.load_images(sampled_df['path'].values,
self.dataset.input_shape)
if self.data_type == Dataset.TEXT_TYPE:
x_data = sampled_df['text'].values
scores = self.model.score(x_data)
entropy_func = lambda scores: np.sum(scores * np.log(1 / scores),
axis=-1)
if type(scores) == list:
entropy = np.array(
[entropy_func(score).mean() for score in scores])
else:
entropy = entropy_func(scores)
assert len(entropy.shape) == 1
num = min(size, len(entropy) - 1)
if reverse_entropy:
entropy_indexes = np.argpartition(entropy, num)[:num]
else:
entropy_indexes = np.argpartition(-entropy, num)[:num]
response = (
sampled_df.iloc[entropy_indexes],
current_stage,
x_data[entropy_indexes],
entropy[entropy_indexes].tolist(),
)
return response
def add_label(self, _id, label):
# Validate label
# TODO: Reevaluate this get_data thing, I'm not a fan of this.
data = self.dataset.get_data(_id)
self.label_helper.validate(data, label)
label = self.label_helper.decode(label)
# _id is just the path to the file
self.dataset.add_label(_id,
label,
self.dataset.current_stage,
user=self.user)
@property
def title(self):
return self.task.title
@property
def description(self):
return self.task.description
def threaded_train(self):
self.trainer.train()
def threaded_label(self):
self.labeller.start()
def get_history(self):
return self.trainer.get_history()
def get_stats(self):
return self.dataset.stats
import numpy
import itertools
from training.dataprep.airbnb_example import AirbnbExample
from training.trainer import Trainer
from training.model import DocLabelingModel
example = AirbnbExample(file_path="/path/to/file.csv")
texts = numpy.array([i for i in example.texts()])
labels = numpy.array([i for i in example.labels()])
eval_train_index = len(texts) - 50
train_texts = texts[:eval_train_index]
eval_texts = texts[eval_train_index:]
train_labels = labels[:eval_train_index]
eval_labels = labels[eval_train_index:]
model = DocLabelingModel(len(labels[0]))
trainer = Trainer(model.model)
model = trainer.train(train_texts, train_labels)
print(trainer.evaluate(eval_texts, eval_labels))
# simple experiment
predictions = model.predict(eval_texts)
results = [(example.read_prediction(i), example.read_prediction(j))
for (i, j) in zip(predictions, eval_labels)]
for result in results:
print(result, "\n")
FaceLandmarkDataset([{'root_dir': config.TRAIN_DATA_DIR,
'label_file': config.LANDMARKS_ANNO_FILE}],
point_num=config.NUM_LANDMARKS,
transform=transform),
batch_size = config.batch_size,
num_workers = config.num_threads,
shuffle=True)
model = LNet()
model.load_state_dict(torch.load('result/iris_lnet/check_point/32_landmarks_model_200.pth'))
if torch.cuda.device_count() > 1:
print("Train on ", torch.cuda.device_count(), " GPUs")
nn.DataParallel(model)
model.to(device)
lossfn = nn.MSELoss()
checkpoint = CheckPoint(config.save_path)
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
# optimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=config.step, gamma=0.1)
logger = Logger(config.save_path)
trainer = Trainer(config.learning_rate, train_loader, model, optimizer,
lossfn, scheduler, logger, device, config.save_path)
for epoch in range(1, config.nEpochs+1):
trainer.train(epoch)
checkpoint.save_model(model, index=epoch, tag=str(config.NUM_LANDMARKS)+'_landmarks')
class LabelApp:
@staticmethod
def load_from(config_path):
with open(config_path) as f:
config = yaml.load(f)
parser = ConfigParser(config)
parser._create_directories()
task = Task.load_from(parser.task)
dataset = Dataset.load_from(parser.dataset)
model_config = config['model']
label_helper = Label.load_from(parser.label)
user = config['user']
return LabelApp(task, dataset, label_helper, user, model_config,
parser)
def __init__(self,
task,
dataset,
label_helper,
user,
model_config,
config,
model_labelling=True):
self.config = config.config
self.task = task
self.dataset = dataset
self.data_type = self.dataset.data_type
self.label_helper = label_helper
model_directory = model_config['directory']
self.model = model_builder.ModelBuilder(dataset, self.label_helper,
model_config).build()
self.trainer = Trainer(model_directory,
self.model,
self.dataset,
self.label_helper,
logger=logger)
self.trainer.load_existing()
self.labeller = ModelLabeller(self.model,
self.dataset,
self.label_helper,
logger=logger)
self.user = user
self.model_labelling = model_labelling
def score(self, x):
scores = self.model.score(x)
return scores
def predict(self, x):
predictions = self.model.predict(x)
return predictions
@property
def is_done(self):
return len(self.dataset.unlabelled) == 0
def next_model_labelled_batch(self, size=100):
model_labelled, target_class = self.dataset.model_labelled(size)
return model_labelled, target_class
def next_batch(self,
size=10,
force_stage=None,
reverse_entropy=False,
prediction=False):
if self.is_done:
raise ValueError(
"Tried to sample a batch when there is nothing else to sample")
logger.debug("Sampling a batch for {} set.".format(
self.dataset.current_stage))
self.dataset.set_current_stage()
current_stage = force_stage if force_stage else self.dataset.current_stage
if current_stage == Dataset.TEST:
sampled_df = self.dataset.sample(size)
return sampled_df, current_stage, [], [0.5] * len(
sampled_df) # TODO: This needs to be fixed
# Generate training data
sampled_df = self.dataset.sample(size * 5)
if self.data_type == Dataset.IMAGE_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
if self.data_type == Dataset.TEXT_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
if self.data_type == Dataset.JSON_TYPE:
x_data, ids = self.dataset.unlabelled_set(size * 5)
scores = self.model.score(x_data)
entropy_func = lambda scores: np.sum(scores * np.log(1 / scores),
axis=-1)
if len(scores.shape) == 3:
entropy = np.array(
[entropy_func(score).mean() for score in scores])
else:
entropy = entropy_func(scores)
assert len(entropy.shape) == 1
num = min(size, len(entropy) - 1)
if reverse_entropy:
entropy_indexes = np.argpartition(entropy, num)[:num]
else:
entropy_indexes = np.argpartition(-entropy, num)[:num]
# Make predictions
# TODO: This doesn't work for text or json types
if self.data_type == Dataset.IMAGE_TYPE:
x_to_score = x_data[entropy_indexes]
else:
x_to_score = []
y_prediction = None
if prediction and len(x_to_score) > 0:
y_prediction = self.predict(x_to_score)
response = (
sampled_df.iloc[entropy_indexes],
current_stage,
y_prediction,
entropy[entropy_indexes].tolist(),
)
return response
def add_labels(self, labels, avg_time_taken):
is_classification = self.label_helper.label_type == 'classification'
if is_classification:
is_binary_classification = len(self.label_helper.classes) == 2
for idx, label in enumerate(labels):
_id = label['path']
is_target_class = label['is_target_class']
save = idx == len(labels) - 1
if is_target_class:
self.dataset.add_label(
label['path'],
label['target_class'],
Dataset.TRAIN,
user=self.user,
save=save,
time_taken=avg_time_taken,
)
else:
if is_binary_classification:
self.dataset.add_label(
label['path'],
0 if label['target_class'] == 1 else 1,
Dataset.TRAIN,
user=self.user,
save=save,
time_taken=avg_time_taken,
)
else:
# If the task is not binary classification, then its impossible to know what the "other" label is.
# Flag this as USER_MODEL_DISAGREEMENT
self.dataset.add_label(
label['path'],
label['target_class'],
Dataset.USER_MODEL_DISAGREEMENT,
user=self.user,
save=save,
time_taken=avg_time_taken,
)
else:
# TODO: The is_classification case should fit nicely into code like the ones below: please refactor
for label in labels:
self.dataset.add_label(
label['path'],
label['label'],
Dataset.TRAIN,
user=self.user,
time_taken=avg_time_taken,
)
def add_label(self, _id, label, time_taken):
# Validate label
# TODO: Reevaluate this get_data thing, I'm not a fan of this.
data = self.dataset.get_data(_id)
self.label_helper.validate(data, label)
label = self.label_helper.decode(label)
# _id is just the path to the file
self.dataset.add_label(
_id,
label,
self.dataset.current_stage,
user=self.user,
save=True,
time_taken=time_taken,
)
@property
def title(self):
return self.task.title
@property
def description(self):
return self.task.description
@property
def template(self):
return self.task.template
def threaded_train(self):
self.trainer.train()
def threaded_label(self):
self.labeller.start()
def get_history(self):
return self.trainer.get_history()
def get_stats(self):
return self.dataset.stats
def main():
logger = Logger()
logger.log("Starting CNP")
args = argumenter.parse_arguments(logger)
if abs(args.test_size - -1.0) < 0.01 and args.abs_test_size == -1:
raise Exception("At least one of test size parameters must be set")
if abs(args.test_size - -1.0) >= 0.01 and args.abs_test_size != -1:
raise Exception("Only one of test size parameters should be set")
test_size = args.abs_test_size
if args.abs_test_size == -1:
test_size = args.test_size
random_seed = args.random_seed
if not random_seed:
random_seed = random.randint(1, 2_000_000_000)
random.seed(a=random_seed)
logger.log(f"Using seed={random_seed}")
logger.log("Init Text Processor")
text_processor = TextProcessor("data/APRC/{}".format(args.data_file),
"data/embeddings/wiki-news-300d-1M.vec",
test_size=test_size,
sents_limit=args.sent_count,
rare_word_threshold=args.rare_threshold,
logger=logger)
logger.log("Init Train Dataset")
train_dataset = DatasetNonContextual(
sents=text_processor.train_sents,
max_seq_len=text_processor.max_seq_len,
mask_ratios=args.mask_ratios,
random_every_time=args.dataset_random_every_time,
to_cuda=args.to_cuda)
logger.log("Init Test Datasets")
test_datasets = []
test_datasets.append(
DatasetNonContextual(sents=text_processor.test25,
max_seq_len=text_processor.max_seq_len,
mask_ratios=[.25],
to_cuda=args.to_cuda))
test_datasets.append(
DatasetNonContextual(sents=text_processor.test50,
max_seq_len=text_processor.max_seq_len,
mask_ratios=[0.5],
to_cuda=args.to_cuda))
logger.log("Vocab size: ", len(text_processor.id2w))
logger.log("Init model")
model = CNP(enc_hidden_layers=args.enc_layers,
dec_hidden_layers=args.dec_layers,
emb_weight=text_processor.embedding_matrix,
max_seq_len=text_processor.max_seq_len,
use_weight_matrix=args.use_weight_matrix,
dropout=args.dropout,
use_latent=args.use_latent,
nheads=args.number_of_heads,
normalize_weights=args.normalize_weights,
to_cuda=args.to_cuda)
logger.log("Model has {} parameters".format(
sum(p.numel() for p in model.parameters() if p.requires_grad)))
logger.log("Init Trainer")
# assume every test_ds has only one mask ration
tags = [test_ds.mask_ratios[0] for test_ds in test_datasets]
trainer = Trainer(model=model,
train_dataset=train_dataset,
test_datasets=test_datasets,
tags=tags,
batch_size=args.batch_size,
opt=args.opt,
learning_rate=args.learning_rate,
momentum=args.momentum,
epoch_count=args.epochs,
acc_topk=args.topk,
kl_weight=args.kl_weight,
print_interval=args.print_interval,
bleu_sents=text_processor.bleu_sents,
to_cuda=args.to_cuda,
logger=logger,
id2w=text_processor.id2w)
logger.log("Start training")
train_loss_per_epoch, test_losses_per_epoch = trainer.run()
plotter = Plotter(train_loss_per_epoch, test_losses_per_epoch, tags,
logger)
plotter.plot()
criterion = L1SparseConsistencyLoss()
model.set_criterion(criterion)
optimizer = tfac.get_optimizer(config)(model.parameters(), lr=config.lr)
dataset_splits = dfac.get_dataset(config)
dataloader_train = DataLoader(dataset_splits['train'],
batch_size=config.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True)
dataloaders = {}
for split in ['train_valid', 'valid', 'test']:
if len(dataset_splits[split]) > 0:
dataloaders[split] = DataLoader(dataset_splits[split],
batch_size=1,
shuffle=False,
pin_memory=True)
f = open(os.path.join(directory, 'train.txt'), 'a')
sys.stdout = Logger(sys.stdout, f)
model.cuda()
model.train()
with Trainer(model, dataloader_train, dataloaders, optimizer, model_dir,
config) as trainer:
trainer.train()
def core_flow(config: MutableMapping) -> None:
dataset = DatasetCollection(config)
trainer = Trainer(dataset, config)
if config.experiment.inference_ckpt:
# testing mode takes precedence of training if both ckpts specified
logger.info(
f'Testing model weights loaded from {config.experiment.inference_ckpt}...'
)
trainer.init_test(config.experiment.inference_ckpt)
elif config.trainer.restart_training_ckpt:
# restarting training takes precedence over just building custom swa checkpoints
logger.info(
f'Restarting model training from {config.trainer.restart_training_ckpt}...'
)
trainer.train(config.trainer.restart_training_ckpt)
elif config.trainer.build_swa_from_ckpts:
logger.info(
f'Building swa checkpoint from specified ckpts: {config.trainer.build_swa_from_ckpts}...'
)
swa_ckpt = trainer.swa_ckpt_build(
mode="custom", ckpt_list=config.trainer.build_swa_from_ckpts)
logger.info(
f'Successfully built SWA checkpoint ({swa_ckpt}) from provided list of checkpoints, '
f'proceeding with test')
trainer.init_test(swa_ckpt)
else:
logger.info('Starting model training from scratch...')
trainer.train()
print("Done")
else:
print("Loading and cleaning data")
dataset = dataset_map[args.dataset](do_clean=True)
# Split the data
dataset.split_data(
dataset_ratio=args.split_ratio,
test_size=args.test_ratio)
if args.save_data:
print("Saving cleaned data")
dataset.data.to_csv(os.path.join(config.model_dir, args.dataset+'_clean.csv'), index=False)
print("Done")
trainer = Trainer(dataset=dataset, models=args.models, transforms=args.feats, cfg=config, grid=False)
# If we only want to test
if args.test_only:
trainer.load_model(args.models[0], args.feats[0], args.load_path)
# Train
if not(args.test_only):
trainer.train()
# Get the evaluation metric
# If test only GridSearchCV is not fitted yet -> so set grid to False
if args.test_only:
metrics = trainer.evaluate()
else:
metrics = trainer.evaluate()
# Save results
'Predictions from an old simulation. The configuration is uploaded...'
)
print('Number of epochs:' + str(epochs))
print('Latent state dim:' + str(model_config['dim_x']))
time.sleep(5)
train = False
retrain = False
model_config.update({'ds': ds_sel})
# evaluation
output_sel = Outputs # can create new class deriving from it if need richer outputs
#
# Run
#
# load
outputs = output_sel(out_dir)
ds = ds_sel(seq_len, seq_stride, in_dir)
outputs.set_ds(ds)
model = model_sel(ds.dim_u, ds.dim_y, model_config)
outputs.set_model(model, predict_len, model_config['dim_x'])
# train
if train:
trainer = Trainer(model, model_dir)
trainer.train(ds, epochs, retrain=retrain, test_data=test_data)
outputs.set_trainer(trainer)
# evaluate
outputs.create_all()
from functools import partial
import yaml
from joblib import cpu_count
from torch.utils.data import DataLoader
import torch.backends.cudnn as cudnn
from training.data import get_datasets
from training.trainer import Trainer
cudnn.benchmark = True
if __name__ == '__main__':
with open("config/train.yaml", "r") as f:
config = yaml.load(f)
batch_size = config.pop('batch_size')
get_dataloader = partial(DataLoader,
batch_size=batch_size,
num_workers=cpu_count(),
shuffle=True,
drop_last=True,
pin_memory=True)
datasets = get_datasets(config['dataset'])
train, val = map(get_dataloader, datasets)
trainer = Trainer(config, train=train, val=val)
trainer.train()
def setUp(self):
self.q_file = '../data/query_groups.txt'
self.i_file = '../data/input_groups_2.txt'
self.tr = Trainer()
search_name = config["model"]["architecture"]
model = getattr(importlib.import_module(search_name), 'model')(config)
model.cuda()
datasets = {}
dataset_splits = DatasetSplitter.generate_splits(config)
transformations = TransformsGenerator.get_final_transforms(config)
for key in dataset_splits:
path, batching_config, split = dataset_splits[key]
transform = transformations[key]
datasets[key] = VideoDataset(path, batching_config, transform, split)
trainer = Trainer(config, model, datasets["train"], logger)
evaluator = Evaluator(config,
datasets["validation"],
logger,
action_sampler=None,
logger_prefix="validation")
# Resume training
try:
trainer.load_checkpoint(model)
except Exception as e:
logger.print(e)
logger.print("Cannot play without loading checkpoint")
exit(1)
model.eval()
def train(architecture_name, test_run, only_eval, overfit):
trainer = Trainer.from_config(architecture_name=architecture_name)
trainer.train(test_run=test_run, only_eval=only_eval, overfit=overfit)