def main(): # capture the config path from the run arguments # then process the json configuration file try: args = get_args() json_file = '../configs/example.json' # config = process_config(args.config) config = process_config(json_file) except: print("missing or invalid arguments") exit(0) # create the experiments dirs create_dirs([config.summary_dir, config.checkpoint_dir]) # create tensorflow session sess = tf.compat.v1.Session() # create your data generator data = DataGenerator(config) data.generate_data() # create an instance of the model you want model = ExampleModel(config) # create tensorboard logger logger = Logger(sess, config) # create trainer and pass all the previous components to it trainer = ExampleTrainer(sess, model, data, config, logger) #load model if exists model.load(sess) # here you train your model trainer.train() # here you evaluate your model evaluator = Evaluator(trainer.sess, trainer.model, data, config, logger) evaluator.evaluate() evaluator.analysis_results()
def main(): # capture the config path from the run arguments # then process the json configuration file config = fetch_args() # create the experiments dirs create_dirs([config.tensorboard_log_dir, config.checkpoint_dir]) print('Create the data generator.') train_data_loader = DataLoader(config, 'train') valid_data_loader = DataLoader(config, 'valid') print('Create the model.') model = Model(config) if config.pretrained_model_checkpoint is not None: model.load(config.pretrained_model_checkpoint) if config.evaluate: print('Predicting on test set.') test_data_loader = DataLoader(config, 'test') evaluator = Evaluator(model.model, test_data_loader, config) evaluator.evaluate() exit(0) print('Create the trainer') trainer = Trainer(model.model, train_data_loader, valid_data_loader, config) print('Start training the model.') trainer.train()
def evaluate(models, data_set_path, log_save_path, measurement, test_times=1): data_set_info_file = open(f"{data_set_path}\\data_set_info.json", 'r') data_set_info = json.load(data_set_info_file) if not os.path.exists(f"{log_save_path}\\{data_set_info['name']}"): os.mkdir(f"{log_save_path}\\{data_set_info['name']}") shutil.copy(f"{data_set_path}\\data_set_info.json", f"{log_save_path}\\{data_set_info['name']}") stream_list = [path for path in os.listdir(data_set_path) if os.path.isdir(f"{data_set_path}\\{path}")] stream_list.sort(key=lambda s: int(s.split('_')[1])) # stream_number records_list = [] for stream in stream_list: log_stream_path = f"{log_save_path}\\{data_set_info['name']}\\{stream}" if not os.path.exists(log_stream_path): os.mkdir(log_stream_path) if not os.path.exists(f"{log_stream_path}\\record"): os.mkdir(f"{log_stream_path}\\record") if not os.path.exists(f"{log_stream_path}\\figure"): os.mkdir(f"{log_stream_path}\\figure") data = pd.read_csv(f"{data_set_path}\\{stream}\\data.csv", header=None) with open(f"{data_set_path}\\{stream}\\data_info.json", 'r') as data_info_file: data_info = json.load(data_info_file) X = np.array(data.iloc[:, 1:]) y = np.array(data.iloc[:, 0]) perf_records = {} for model in models: for _ in range(test_times): test_model = copy.deepcopy(models[model]) test_model.budget = data_info['budget'] evaluator = Evaluator(measurement=measurement, pretrain_size=1, batch_size=1, budget=data_info['budget']) if model not in perf_records.keys(): perf_records[model] = evaluator.evaluate(X, y, model=test_model) else: perf_records[model] += evaluator.evaluate(X, y, model=test_model) perf_records[model] = perf_records[model] / test_times perf_records = pd.DataFrame(perf_records) records_list.append(perf_records) perf_records.to_csv(f"{log_stream_path}\\record\\{measurement}.csv", index=None) plot_lines(perf_records, f"{log_stream_path}\\figure\\{measurement}", "pdf", 15, 'time', measurement) report_file = open(f"{log_save_path}\\report.md", 'a') report_file.write(f"# {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}\n") report_file.write(f"{data_set_info}\n\n") report_file.write(f"{measurement}\n\n") table = get_report(data_set_info, records_list, file_type="md") report_file.write(f"{table}\n") report_file.close()
def main(config: DictConfig) -> None: print(OmegaConf.to_yaml(config)) torch.manual_seed(config.eval.seed) torch.cuda.manual_seed_all(config.eval.seed) np.random.seed(config.eval.seed) random.seed(config.eval.seed) use_cuda = config.eval.cuda and torch.cuda.is_available() device = torch.device('cuda' if use_cuda else 'cpu') char2id, id2char = load_label(config.eval.label_path, config.eval.blank_id) audio_paths, transcripts, _, _ = load_dataset(config.eval.dataset_path, config.eval.mode) test_dataset = SpectrogramDataset( config.eval.audio_path, audio_paths, transcripts, config.audio.sampling_rate, config.audio.n_mel, config.audio.frame_length, config.audio.frame_stride, config.audio.extension, config.train.sos_id, config.train.eos_id, ) test_loader = AudioDataLoader( test_dataset, batch_size=config.eval.batch_size, num_workers=config.eval.num_workers, ) model = load_test_model(config, device) print('Start Test !!!') evaluator = Evaluator(config, device, test_loader, id2char) evaluator.evaluate(model)
class Trainer(object): """Trainer Class""" def __init__(self, optimizer, criterion, batch_size, device): self.optimizer = optimizer self.criterion = criterion self.batch_size = batch_size self.device = device self.evaluator = Evaluator(criterion=self.criterion) def _train_batch(self, model, iterator, iteratorQuery, teacher_ratio, clip): model.train() epoch_loss = 0 for _, batch in enumerate(zip(iterator, iteratorQuery)): batch_ques, batch_query = batch src_ques, src_len_ques = batch_ques.src src_query, src_len_query = batch_query.src trg = batch_query.trg self.optimizer.zero_grad() input_trg = trg if model.name == RNN_NAME else trg[:, :-1] output = model(src_ques, src_len_ques, src_query, src_len_query, input_trg, teacher_ratio) trg = trg.t() if model.name == RNN_NAME else trg[:, 1:] output = output.contiguous().view(-1, output.shape[-1]) trg = trg.contiguous().view(-1) # output: (batch_size * trg_len) x output_dim # trg: (batch_size * trg_len) loss = self.criterion(output, trg) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), clip) self.optimizer.step() epoch_loss += loss.item() length = len(iterator) return epoch_loss / len(iterator) def _get_iterators(self, train_data, valid_data, model_name): return BucketIterator.splits((train_data, valid_data), repeat=False, batch_size=self.batch_size, sort_within_batch=False, sort_key=lambda x: len(x.src), device=self.device) def _epoch_time(self, start_time, end_time): elapsed_time = end_time - start_time elapsed_mins = int(elapsed_time / 60) elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) return elapsed_mins, elapsed_secs def _log_epoch(self, train_loss, valid_loss, epoch, start_time, end_time): minutes, seconds = self._epoch_time(start_time, end_time) print(f'Epoch: {epoch+1:02} | Time: {minutes}m {seconds}s') print( f'\tTrain Loss: {train_loss:.3f} | Train PPL: {np.exp(train_loss):7.3f}' ) print( f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {np.exp(valid_loss):7.3f}' ) def _train_epoches(self, model, train_data, train_data_query, valid_data, valid_data_query, num_of_epochs, teacher_ratio, clip): best_valid_loss = float('inf') # pylint: disable=unbalanced-tuple-unpacking train_iterator, valid_iterator = self._get_iterators( train_data, valid_data, model.name) train_iterator_query, valid_iterator_query = self._get_iterators( train_data_query, valid_data_query, model.name) for epoch in range(num_of_epochs): start_time = time.time() train_loss = self._train_batch(model, train_iterator, train_iterator_query, teacher_ratio, clip) valid_loss = self.evaluator.evaluate(model, valid_iterator, valid_iterator_query, teacher_ratio) end_time = time.time() self._log_epoch(train_loss, valid_loss, epoch, start_time, end_time) if valid_loss < best_valid_loss: best_valid_loss = valid_loss Chechpoint.save(model) def train(self, model, train_data, train_data_query, valid_data, valid_data_query, num_of_epochs=20, teacher_ratio=1.0, clip=1): """Train model""" self._train_epoches(model, train_data, train_data_query, valid_data, valid_data_query, num_of_epochs, teacher_ratio, clip)
class Experiment(): dataset = None dataset_switch = None model_manager = None switch_manager = None evaluator = None force = None def __init__(self, dataset_id, dataset_switch_id, force): self._init_dir() self.dataset = DataSet(dataset_id = dataset_id, sframe = True) self.dataset_switch = DataSet(dataset_id = dataset_switch_id, sframe = False) self.model_manager = ModelManager() self.switch_manager = SwitchManager() self.evaluator = Evaluator() self.force = force def _init_dir(self): import os from settings import DIR for d in DIR: if not os.path.exists(d): os.makedirs(d) def _train_rec_models(self): self.model_manager.train_models(dataset = self.dataset) def _test_rec_models(self): self.model_manager.test_models(dataset = self.dataset) def _evaluate_rec_models(self): self.model_manager.evaluate_models(dataset = self.dataset) def _create_datasets_switch(self): self.dataset_switch.prepare_switch_dataset(dataset = self.dataset, model_manager = self.model_manager, force = self.force) def _train_switch(self): self.switch_manager.train_models(dataset_switch = self.dataset_switch, force = self.force) def _test_switch(self): self.switch_manager.rating_prediction_switches(dataset = self.dataset, dataset_switch = self.dataset_switch, model_manager = self.model_manager, force = self.force) def _evaluate(self): self.evaluator.evaluate(dataset = self.dataset, dataset_switch = self.dataset_switch, model_manager = self.model_manager, switch_manager = self.switch_manager, force = True) def run(self): self._train_rec_models() self._test_rec_models() self._evaluate_rec_models() self._create_datasets_switch() self._train_switch() self._test_switch() self._evaluate()
lang = Lang() trans = U.tokenizeTranscripts('train') lang.init_lang(trans) output_size = lang.num_items dev_dataset = SpeechDataset(lang, 'dev') dev_dataloader = SpeechDataLoader(dev_dataset, batch_size=batch_size) num_layers = 3 hidden_size = 256 input_size = 40 key_size = 128 value_size = 128 bidirectional = True p = 3 embedding_size = 128 encoder = EncoderRNN(input_size, hidden_size, key_size, value_size, num_layers, bidirectional, p) decoder = DecoderRNN(output_size, embedding_size, hidden_size, key_size, value_size, num_layers) teacher_forcing_ratio = 1.0 las = LAS(encoder, decoder, teacher_forcing_ratio) criterion = nn.CrossEntropyLoss(size_average=False, ignore_index=C.PAD_TOKEN_IDX) evaluator = Evaluator(criterion) evaluator.evaluate(las, dev_dataloader)
max_len, hidden_size, bidirectional=bidirectional, variable_lengths=True) dual_encoder = DualEncoder(context_encoder, response_encoder) if torch.cuda.is_available(): dual_encoder.cuda() for param in dual_encoder.parameters(): param.data.uniform_(-0.08, 0.08) # train t = SupervisedTrainer(loss_func=loss_func, batch_size=1, checkpoint_every=30, print_every=100, expt_dir=opt.expt_dir) t.train(dual_encoder, train, batch_size=1, num_epochs=20, dev_data=dev, optimizer=optimizer, resume=opt.resume) evaluator = Evaluator(batch_size=1) l, precision, recall = evaluator.evaluate(dual_encoder, dev) print("Precision: {}, Recall: {}".format(precision, recall))
class SupervisedTrainer(object): def __init__(self, export_dir='experiment', loss=NLLLoss(), batch_size=64, random_seed=None, checkpoint_every=100, print_every=100): self._trainer = "Simple Trainer" self.random_seed = random_seed if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) self.loss = loss self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size) self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every if not os.path.isabs(export_dir): export_dir = os.path.join(os.getcwd(), export_dir) self.export_dir = export_dir if not os.path.exists(self.export_dir): os.makedirs(self.export_dir) self.batch_size = batch_size self.logger = logging.getLogger(__name__) def _train_batch(self, input_variable: torch.Tensor, input_lengths, target_variable, model, teacher_model=None, teacher_forcing_ratio=.5): loss = self.loss loss.reset() input_batch_size = input_variable.size(0) input_seq_length = input_variable.size(1) target_batch_size = target_variable.size(0) target_seq_length = target_variable.size(1) if teacher_model is not None: tgt_arr = np.array([[model.sos_id]] * batch_size, 'i') tgt_var = torch.autograd.Variable( torch.LongTensor(tgt_arr).type(torch.LongTensor)) transformer_output = model(input_variable, tgt_var) # transformer_output [batch_size,1,tgt_vocab_size] teacher_output, teacher_hidden, other = teacher_model( target_variable, target_variable.size(1)) else: input_var = input_variable.view(input_seq_length, input_batch_size, 1) target_var = target_variable.view(target_seq_length, target_batch_size, 1) aacc = input_lengths.size() decoder_output, other = model(input_var, target_var, input_lengths) for step, step_output in enumerate(decoder_output): if teacher_model is not None: # step_output = tgt = target_variable[:, step + 1].contiguous().view(-1) #f**k = teacher_output[-step].permute(2,1,0) loss.eval_batch(step_output, tgt, teacher_output[-step]) else: liner_func = nn.Linear(128, 8) dec_out = liner_func(step_output) dec_out: torch.Tensor = F.log_softmax(dec_out, dim=1) loss.eval_batch(dec_out.view(input_batch_size, -1), target_variable[:, step + 1]) model.zero_grad() loss.backward() self.optimizer.step() return loss.get_loss() def _train_epoches(self, data, model, teacher_model, n_epochs, start_epoch, start_step, dev_data, teacher_forcing_ratio=0): log = self.logger print_loss_total = 0 epoch_loss_total = 0 device = None if torch.cuda.is_available() else -1 batch_iterator = torchtext.data.BucketIterator( dataset=data, batch_size=self.batch_size, sort=False, sort_within_batch=True, sort_key=lambda x: len(x.src), device=device, repeat=False) step_per_epoch = len(batch_iterator) total_steps = step_per_epoch * n_epochs step = start_step step_elapsed = 0 for epoch in range(start_epoch, n_epochs + 1): log.debug("Epoch: %d, Step: %d" % (epoch, step)) batch_generator = batch_iterator.__iter__() for _ in range((epoch - 1) * step_per_epoch, step): next(batch_generator) model.train(True) for batch in batch_generator: step += 1 step_elapsed += 1 input_var, input_length = getattr(batch, 'src') target_var = getattr(batch, 'tgt') loss = self._train_batch(input_variable=input_var, input_lengths=input_length, target_variable=target_var, model=model, teacher_model=teacher_model) print_loss_total += loss epoch_loss_total += loss if step % self.print_every == 0 and step_elapsed > self.print_every: print_loss_avg = print_loss_total / self.print_every print_loss_total = 0 log_msg = 'Progress: %d%%, Train %s: %.4f' % ( step / total_steps * 100, self.loss.name, print_loss_avg) log.info(log_msg) # Checkpoint if step % self.checkpoint_every == 0 or step == total_steps: Checkpoint(model=model, optimizer=self.optimizer, epoch=epoch, step=step, input_vocab=data.fields['src'].vocab, output_vocab=data.fields['tgt'].vocab).save( self.export_dir) if step_elapsed == 0: continue epoch_loss_avg = epoch_loss_total / \ min(step_per_epoch, step - start_step) epoch_loss_total = 0 log_msg = "Finished epoch %d: Train %s: %.4f" % ( epoch, self.loss.name, epoch_loss_avg) if dev_data is not None: dev_loss, accuracy = self.evaluator.evaluate(model, dev_data) self.optimizer.update(dev_loss, epoch) log_msg += ", Dev %s: %.4f, Accuracy: %.4f" % ( self.loss.name, dev_loss, accuracy) model.train(mode=True) else: self.optimizer.update(epoch_loss_avg, epoch) log.info(log_msg) def train(self, model, data, teacher_model=None, num_epochs=5, resume=False, dev_data=None, optimizer=None, teacher_forcing_ratio=0): if resume: latest_checkpoint_path = Checkpoint.get_latest_checkpoint( self.expt_dir) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model self.optimizer = resume_checkpoint.optimizer resume_optim = self.optimizer.optimizer defaults = resume_optim.param_groups[0] defaults.pop('param', None) defaults.pop('initial_lr', None) self.optimizer.optimizer = resume_optim.__class__( model.parameters(), **defaults) start_epoch = resume_checkpoint.epoch step = resume_checkpoint.step else: start_epoch = 1 step = 0 if optimizer is None: optimizer = Optimizer(optim.Adam(model.parameters()), max_grad_norm=5) self.optimizer = optimizer self.logger.info("Optimizer: %s, Scheduler: %s" % (self.optimizer.optimizer, self.optimizer.scheduler)) self._train_epoches(data, model, teacher_model, num_epochs, start_epoch, step, dev_data=dev_data, teacher_forcing_ratio=teacher_forcing_ratio) return model
import os from options.test_options import TestOptions from evaluator import train_classifier from evaluator.evaluator import EvaluatorDataset from evaluator.evaluator import Evaluator from evaluator.dataset import ClassifierDataset from torch.utils.data import DataLoader if __name__=="__main__": opt = TestOptions().parse() classifier_path = 'evaluator/checkpoints/latest_{}_resnet.pth'.format(opt.evaluate_mode) if not os.path.exists(classifier_path): train_classifier(mode=opt.evaluate_mode) training_data = ClassifierDataset(opt.evaluate_mode) dataset = EvaluatorDataset(opt) dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=4) evaluator = Evaluator(opt, num_classes=training_data.num_classes, text2label=training_data.text2label) for data in dataloader: try: evaluator.evaluate(data) evaluator.record_current_results() except: print('!!!error!!! pass current data') evaluator.compute_final_results()
class Trainer: def __init__(self, criterion): super(Trainer, self).__init__() self.criterion = criterion self.evaluator = Evaluator(criterion) def _train_batch(self, model, input_variables, input_lengths, target_variables): decoder_outputs, ret_dict = model(input_variables, input_lengths, target_variables) acc_loss = self.criterion(decoder_outputs.contiguous(), target_variables[1:, :].contiguous()) acc_loss = acc_loss.view( target_variables.size(0) - 1, target_variables.size(1)) acc_loss = acc_loss.sum(0).mean() self.optimizer.zero_grad() acc_loss.backward() params = itertools.chain.from_iterable( [group['params'] for group in self.optimizer.param_groups]) torch.nn.utils.clip_grad_norm_(params, max_norm=self.max_grad_norm) self.optimizer.step() return acc_loss.data.item() def train(self, train_dataloader, dev_dataloader, model, lr, num_epochs): self.max_grad_norm = 5 self.optimizer = torch.optim.Adam(model.parameters(), lr=lr) for epoch in range(9, num_epochs): model.train(True) epoch_loss = 0 num_batches = len(train_dataloader.dataloader) for (batch_idx, data) in enumerate(train_dataloader.dataloader): input_variables, input_lengths, target_variables, target_lengths = data input_variables = U.var( torch.from_numpy(input_variables).float()) target_variables = U.var( torch.from_numpy(target_variables).long()) input_variables = input_variables.transpose(0, 1) target_variables = target_variables.transpose(0, 1) batch_loss = self._train_batch(model, input_variables, input_lengths, target_variables) epoch_loss += batch_loss if batch_idx % 50 == 0: print("batch %d avg_loss %f" % (batch_idx, epoch_loss / (batch_idx + 1))) print("epoch %d train_epoch_loss %f" % (epoch, epoch_loss / num_batches)) if epoch % 1 == 0: U.checkpoint(epoch, model) val_epoch_loss = self.evaluator.evaluate(model, dev_dataloader) print("epoch %d val_epoch_loss %f" % (epoch, val_epoch_loss))