def _save_fitable(self, run: Run, fitable: Model): """ :param run: sacred.Run object. see sacred documentation for more details on utility. :param fitable: tensorflow.keras.Model object. """ path = self.exp_config["run_config"]["model_path"] if self.exp_config["run_config"]["save_verbosity"] > 0: fitable.summary() fitable.save(self.exp_config["run_config"]["model_path"]) run.add_artifact(path)
def train(train_corpus: str, dev_corpus: str, c1: float = 0.0, c2: float = 0.0, algorithm: str = 'lbfgs', max_iterations: int = 100, all_possible_transitions: bool = False, window_size: int = 1, model_filename: str = None, _run: Run = None, _log: logger = None): """ running crf experiment """ _run.add_resource(train_corpus) _run.add_resource(dev_corpus) train_sents, _ = get_tagged_sents_and_words(train_corpus) dev_sents, _ = get_tagged_sents_and_words(dev_corpus) X_train = [sent2features(s, window_size) for s in train_sents] y_train = [sent2labels(s) for s in train_sents] X_dev = [sent2features(s, window_size) for s in dev_sents] y_dev = [sent2labels(s) for s in dev_sents] crf = sklearn_crfsuite.CRF( algorithm=algorithm, c1=c1, c2=c2, max_iterations=max_iterations, all_possible_transitions=all_possible_transitions, model_filename=model_filename, ) crf.fit(X_train, y_train) y_pred = crf.predict(X_dev) overall, by_type = evaluate(y_dev, y_pred) _run.info[f'overall_f1'] = overall.f1_score _run.log_scalar('overall_f1', overall.f1_score) _run.info[f'overall_precision'] = overall.precision _run.log_scalar('overall_precision', overall.precision) _run.info[f'overall_recall'] = overall.recall _run.log_scalar('overall_recall', overall.recall) _log.info(f'Overall F1 score: {overall.f1_score}') for _, key in enumerate(sorted(by_type.keys())): for metric_key in by_type[key]._fields: metric_val = getattr(by_type[key], metric_key) _run.info[f'{key}-{metric_key}'] = metric_val _run.log_scalar(f'{key}-{metric_key}', metric_val) _log.info(f'{key}-{metric_key}: {metric_val}') if model_filename is not None: _log.info(f'saving to: {model_filename}.pkl') joblib.dump(crf, f'{model_filename}.pkl') _run.add_artifact(f'{model_filename}.pkl')
def train(train_corpus: str, dev_corpus: str, pacrf: str, model_filename: str, labels: List, c1: float = 0.0, c2: float = 1.0, algorithm: str = 'lbfgs', max_iterations: int = None, all_possible_transitions: bool = False, window_size: int = 0, _run: Run = None, _log: logger = None): """ running crf experiment """ _run.add_resource(train_corpus) _run.add_resource(dev_corpus) train_sents, _ = get_tagged_sents_and_words(train_corpus) dev_sents, _ = get_tagged_sents_and_words(dev_corpus) tmp_train = tempfile.NamedTemporaryFile(mode='w+') # temp_train_corpus = open(f'{model_filename}-{train_corpus}.feature', mode='w+') print_corpus(train_sents, labels, tmp_train, window_size=window_size) # X_dev = [sent2features(s, window_size) for s in dev_sents] y_dev = [sent2labels_colmap(s, col=1) for s in dev_sents] tmp_dev = tempfile.NamedTemporaryFile(mode='w+') # temp_test_corpus = open(f'{model_filename}-{test_corpus}.feature', mode='w+') print_corpus(dev_sents, labels, tmp_dev, window_size=window_size) # to call partial-crf via Popen command # command = f'{pacrf} learn -m {model_filename} -a {algorithm} {temp_train_corpus}' # call([pacrf, "--help"]) crfsuire_proc = Popen([pacrf, "learn", "-m", model_filename, "-a", algorithm, \ "-p", f"c1={c1}", "-p", f"c2={c2}", tmp_train.name]) out, err = crfsuire_proc.communicate() print(out) print(err) # os.system(f'{pacrf} learn -m {model_filename} -a {algorithm} {tmp_train.name}') tmp_train.close() tmp_pred = tempfile.NamedTemporaryFile(mode='w+') # cmd_out([pacrf, "tag", "-m", model_filename, tmp_dev.name, ">", tmp_pred.name]) _run.add_artifact(model_filename) # TODO modified this to call partial-crf via Popen command # y_pred = crf.predict(X_dev) y_pred = get_tagged_sents_and_words(tmp_pred.name) print(y_pred) y_pred = [sent2labels_colmap(s, 0) for s in y_pred] # TODO modified this to read partial-crf via tempfile overall, by_type = evaluate(y_dev, y_pred) tmp_pred.close() tmp_dev.close() _run.info[f'overall_f1'] = overall.f1_score _run.log_scalar('overall_f1', overall.f1_score) _run.info[f'overall_precision'] = overall.precision _run.log_scalar('overall_precision', overall.precision) _run.info[f'overall_recall'] = overall.recall _run.log_scalar('overall_recall', overall.recall) _log.info(f'Overall F1 score: {overall.f1_score}') for _, key in enumerate(sorted(by_type.keys())): for metric_key in by_type[key]._fields: metric_val = getattr(by_type[key], metric_key) _run.info[f'{key}-{metric_key}'] = metric_val _run.log_scalar(f'{key}-{metric_key}', metric_val) _log.info(f'{key}-{metric_key}: {metric_val}')
def sacred_main(_run: Run, seed, showoff, out_dir, batch_size, epochs, tags, model_desc, experiment_id, weights, train_examples, val_examples, deterministic, train_datasets, val_datasets, lr, lr_milestones, lr_gamma, optim_algorithm): seed_all(seed) init_algorithms(deterministic=deterministic) exp_out_dir = None if out_dir: exp_out_dir = path.join(out_dir, experiment_id) makedirs(exp_out_dir, exist_ok=True) print(f'Experiment ID: {experiment_id}') #### # Model #### if weights is None: model = create_model(model_desc) else: details = torch.load(weights) model_desc = details['model_desc'] model = create_model(model_desc) model.load_state_dict(details['state_dict']) model.to(global_opts['device']) print(json.dumps(model_desc, sort_keys=True, indent=2)) #### # Data #### train_loader = create_train_dataloader(train_datasets, model.data_specs, batch_size, train_examples) if len(val_datasets) > 0: val_loader = create_val_dataloader(val_datasets, model.data_specs, batch_size, val_examples) else: val_loader = None #### # Reporting #### reporter = Reporter(with_val=(val_loader is not None)) reporter.setup_console_output() reporter.setup_sacred_output(_run) notebook = None if showoff: title = '3D pose model ({}@{})'.format(model_desc['type'], model_desc['version']) notebook = create_showoff_notebook(title, tags) reporter.setup_showoff_output(notebook) def set_progress(value): if notebook is not None: notebook.set_progress(value) tel = reporter.telemetry tel['config'].set_value(_run.config) tel['host_info'].set_value(get_host_info()) #### # Optimiser #### if optim_algorithm == '1cycle': from torch import optim optimiser = optim.SGD(model.parameters(), lr=0) scheduler = make_1cycle(optimiser, epochs * len(train_loader), lr_max=lr, momentum=0.9) else: scheduler = learning_schedule(model.parameters(), optim_algorithm, lr, lr_milestones, lr_gamma) #### # Training #### model_file = None if exp_out_dir: model_file = path.join(exp_out_dir, 'model-latest.pth') with open(path.join(exp_out_dir, 'config.json'), 'w') as f: json.dump(tel['config'].value(), f, sort_keys=True, indent=2) for epoch in range(epochs): tel['epoch'].set_value(epoch) print('> Epoch {:3d}/{:3d}'.format(epoch + 1, epochs)) def on_train_progress(samples_processed): so_far = epoch * len(train_loader.dataset) + samples_processed total = epochs * len(train_loader.dataset) set_progress(so_far / total) do_training_pass(epoch, model, tel, train_loader, scheduler, on_train_progress) if val_loader: do_validation_pass(epoch, model, tel, val_loader) _run.result = tel['train_pck'].value()[0] if model_file is not None: state = { 'state_dict': model.state_dict(), 'model_desc': model_desc, 'train_datasets': train_datasets, 'optimizer': scheduler.optimizer.state_dict(), 'epoch': epoch + 1, } torch.save(state, model_file) tel.step() # Add the final model as a Sacred artifact if model_file is not None and path.isfile(model_file): _run.add_artifact(model_file) set_progress(1.0) return _run.result