def train(self): os.mkdir(self.paths['path']) if self.use_comet and self.api_key and self.project_name and self.workspace: experiment = Experiment(api_key=self.api_key, project_name=self.project_name, workspace=self.workspace) experiment.log_dataset_hash(self.train_dataset) experiment.add_tags([ str(self.architecture), "text_generation", f"nb_labels_{self.number_labels}" ]) with experiment.train(): hist = self.fit_dataset(self.train_dataset, self.val_dataset, self.epochs) experiment.end() elif self.use_comet: raise Exception( "Please provide an api_key, project_name and workspace for comet_ml" ) else: callbacks = self.callback_func( tensorboard_dir=self.paths['tensorboard_path'], checkpoint_path=self.paths['checkpoint_path']) hist = self.model.fit_dataset(self.train_dataset, self.val_dataset, self.epochs, callbacks) self.metrics = get_metrics(hist, "sparse_categorical_accuracy") self.export_weights(self.model) self.export_info(self.model_info) self.export_metrics(self.metrics) self.export_tokenizer(self.tokenizer) if self.do_zip_model: self.zip_model()
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.add_tags(self.config.tags) experiment.disable_mp() experiment.log_parameters(self.config) self.callbacks.append(experiment.get_keras_callback())
class CometTracker: def __init__(self, comet_params, run_params=None, prev_exp_id=None): if prev_exp_id: # previous experiment api_key = comet_params['api_key'] del comet_params[ 'api_key'] # removing this because the rest of the items need to be passed self.experiment = ExistingExperiment( api_key=api_key, previous_experiment=prev_exp_id, **comet_params) print( f'In CometTracker: ExistingExperiment initialized with id: {prev_exp_id}' ) else: # new experiment self.experiment = Experiment(**comet_params) self.experiment.log_parameters(run_params) def track_metric(self, metric, value, step): self.experiment.log_metric(metric, value, step) def add_tags(self, tags): self.experiment.add_tags(tags) print(f'In [add_tags]: Added these tags to the new experiment: {tags}') def set_name(self, name): self.experiment.set_name(name)
def main(model, optimizer, logger, optimized_function, project_name, work_space, tags, model_config_file, optimizer_config_file, epochs, n_samples, step_data_gen, n_samples_per_dim, reuse_optimizer, reuse_model, shift_model, finetune_model, init_psi): model_config = getattr(__import__(model_config_file), 'model_config') optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') print(optimizer_config) init_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',')]).float().to(device) psi_dim = len(init_psi) model_config['psi_dim'] = psi_dim model_config['n_samples'] = n_samples model_config['n_samples_per_dim'] = n_samples_per_dim optimizer_config['x_step'] = step_data_gen optimized_function_cls = str_to_class(optimized_function) model_cls = str_to_class(model) optimizer_cls = str_to_class(optimizer) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameter('model_type', model) experiment.log_parameter('optimizer_type', optimizer) experiment.log_parameters( {"model_{}".format(key): value for key, value in model_config.items()}) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) # experiment.log_asset("./gan_model.py", overwrite=True) # experiment.log_asset("./optim.py", overwrite=True) # experiment.log_asset("./train.py", overwrite=True) # experiment.log_asset("../model.py", overwrite=True) logger = str_to_class(logger)(experiment) end_to_end_training(epochs=epochs, model_cls=model_cls, optimizer_cls=optimizer_cls, optimized_function_cls=optimized_function_cls, logger=logger, model_config=model_config, optimizer_config=optimizer_config, current_psi=init_psi, n_samples_per_dim=n_samples_per_dim, step_data_gen=step_data_gen, n_samples=n_samples, reuse_optimizer=reuse_optimizer, reuse_model=reuse_model, shift_model=shift_model, finetune_model=finetune_model, experiment=experiment)
def main(logger, optimized_function, optimizer, diff_scheme, optimizer_config_file, project_name, work_space, tags, num_repetitions, n, h, use_true_grad, init_psi, p): device = torch.device('cpu') print("Using device = {}".format(device)) optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') init_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',')]).float().to(device) psi_dim = len(init_psi) optimized_function_cls = str_to_class(optimized_function) optimizer_cls = str_to_class(optimizer) diff_scheme_func = str_to_class(diff_scheme) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameter('optimizer_type', optimizer) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) logger = str_to_class(logger)(experiment) y_model = optimized_function_cls(device=device, psi_init=init_psi) if use_true_grad: ndiff = y_model else: grad_func = lambda f, x, n, h: compute_gradient_of_vector_function( f=f, x=x, n=n, h=h, scheme=diff_scheme_func) ndiff = NumericalDifferencesModel(y_model=y_model, psi_dim=psi_dim, y_dim=1, x_dim=1, n=n, h=h, num_repetitions=num_repetitions, grad_func=grad_func) max_iters = optimizer_config['max_iters'] optimizer_config['max_iters'] = 1 optimizer_config['p'] = p optimizer = optimizer_cls(oracle=ndiff, x=init_psi, **optimizer_config) for iter in range(max_iters): current_psi, status, history = optimizer.optimize() print(current_psi) # if iter % 10 == 0: logger.log_performance(y_sampler=y_model, current_psi=current_psi, n_samples=5000) torch.cuda.empty_cache() logger.log_optimizer(optimizer)
def train_hierarchy_extractor( data_path, bsz, num_workers, lr, weight_decay, warmup_updates, max_updates, accumulation_steps, validation_interval, seed, model_path, tag, device, save_metric, save_min, ): experiment = Experiment(project_name="information-retrieval", auto_output_logging=False) experiment.add_tags([tag]) parameters = { "bsz": bsz, "num_workers": num_workers, "lr": lr, "weight_decay": weight_decay, "warmup_updates": warmup_updates, "max_updates": max_updates, "validation_interval": validation_interval, "seed": seed, "model_path": model_path, "device": device, "accumulation_steps": accumulation_steps, "save_metric": save_metric, "save_min": save_min } experiment.log_parameters(parameters) model = BertHierarchyExtractor(model_path, 12, device) trainer = BertExtractorTrainer( experiment, model, data_path, model_path, bsz, num_workers, lr, weight_decay, warmup_updates, max_updates, accumulation_steps, validation_interval, save_metric, save_min, device, seed, ) try: trainer.train() except Exception as e: raise e
def setup_comet_ml_experiment(api_key, project_name, experiment_name, parameters, tags): """ Function for setting up comet ml experiment """ experiment = Experiment(api_key=api_key, project_name=project_name, auto_metric_logging=False) experiment.set_name(experiment_name) experiment.log_parameters(parameters) experiment.add_tags(tags) return experiment
def main(logger, optimizer, optimized_function, optimizer_config_file, model_config_file, project_name, work_space, tags, reuse_optimizer, init_psi, epochs): model_config = getattr(__import__(model_config_file), 'model_config') optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') current_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',') ]).float().to(device) psi_dim = len(init_psi) print(psi_dim, current_psi) optimized_function_cls = str_to_class(optimized_function) optimizer_cls = str_to_class(optimizer) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) logger = str_to_class(logger)(experiment) y_model = optimized_function_cls(device=device, psi_init=current_psi) model = VoidModel(y_model=y_model, **model_config) optimizer = optimizer_cls(oracle=model, x=current_psi, **optimizer_config) for epoch in range(epochs): if reuse_optimizer: optimizer.update(oracle=model, x=current_psi) else: # find new psi optimizer = optimizer_cls(oracle=model, x=current_psi, **optimizer_config) current_psi, status, history = optimizer.optimize() try: logger.log_optimizer(optimizer) logger.log_grads(model, y_sampler=y_model, current_psi=current_psi, num_repetitions=5000) logger.log_performance(y_sampler=y_model, current_psi=current_psi, n_samples=5000) except Exception as e: print(e) raise
class CometMLMonitor(MonitorBase): """ Send data to https://www.comet.ml. Note: 1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack. 2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze. Therefore the feature is disabled by default. """ def __init__(self, experiment=None, api_key=None, tags=None, **kwargs): """ Args: experiment (comet_ml.Experiment): if provided, invalidate all other arguments api_key (str): your comet.ml API key tags (list[str]): experiment tags kwargs: other arguments passed to :class:`comet_ml.Experiment`. """ if experiment is not None: self._exp = experiment assert api_key is None and tags is None and len(kwargs) == 0 else: from comet_ml import Experiment kwargs.setdefault( 'log_code', True ) # though it's not functioning, git patch logging requires it kwargs.setdefault('auto_output_logging', None) self._exp = Experiment(api_key=api_key, **kwargs) if tags is not None: self._exp.add_tags(tags) self._exp.set_code( "Code logging is impossible because there are too many files ...") self._exp.log_dependency('tensorpack', __git_version__) @property def experiment(self): """ The :class:`comet_ml.Experiment` instance. """ return self._exp def _before_train(self): self._exp.set_model_graph(tf.get_default_graph()) @HIDE_DOC def process_scalar(self, name, val): self._exp.log_metric(name, val, step=self.global_step) def _after_train(self): self._exp.end() def _after_epoch(self): self._exp.log_epoch_end(self.epoch_num)
def _train_with_comet(self, train_dataset, val_dataset): experiment = Experiment(api_key=self.api_key, project_name=self.project_name, workspace=self.workspace) experiment.log_dataset_hash(train_dataset) experiment.add_tags([ str(self.architecture), self.name, f"nb_labels_{self.label_encoder_classes_number}" ]) with experiment.train(): hist = self.fit_dataset(train_dataset, val_dataset) experiment.end() return hist
def make_experiment(env_file, name=None, tags=None): # Get environment values load_dotenv(env_file) COMETML_KEY = os.environ.get("COMETML_KEY") COMETML_PROJECT = os.environ.get("COMETML_PROJECT") # Start and configure experiment experiment = Experiment(COMETML_KEY, COMETML_PROJECT) if name is not None: experiment.set_name(name) if tags is not None: experiment.add_tags(tags) return experiment
class Logger: def __init__(self, send_logs, tags, parameters): self.send_logs = send_logs if self.send_logs: self.experiment = Experiment(api_key="OZwyhJHyqzPZgHEpDFL1zxhyI", project_name="drilling-the-hole", workspace="wwydmanski") self.sent_mb = 0 if self.send_logs: if tags is not None: self.experiment.add_tags(tags) if parameters is not None: self.experiment.log_parameters(parameters) def begin_logging(self, episode_count, steps_per_ep): if self.send_logs: self.experiment.log_parameter("Episode count", episode_count) self.experiment.log_parameter("Max steps per episode", steps_per_ep) def log_round(self, actions, reward, cumulative_reward, angle, loss, step): if self.send_logs: self.experiment.log_metric("Round reward", reward, step=step) self.experiment.log_metric("Per-ep reward", cumulative_reward, step=step) self.experiment.log_metric("Action 1", actions[0], step=step) self.experiment.log_metric("Action 2", actions[1], step=step) self.experiment.log_metric("Current angle", angle, step=step) self.experiment.log_metrics(loss, step=step) def log_episode(self, cumulative_reward, state, step): if self.send_logs: self.experiment.log_metric("Angle", state[0], step=step) self.experiment.log_metric("Goal", state[1], step=step) self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step) def end(self): if self.send_logs: self.experiment.end()
class CometTracker: def __init__(self, comet_params, experiment_name=None, run_params=None): self.experiment = Experiment(**comet_params) if run_params is not None: self.experiment.log_parameters(run_params) if experiment_name is not None: self.experiment.set_name(experiment_name) def track_metric(self, metric, value, step=None): self.experiment.log_metric(metric, value, step) def add_tags(self, tags): self.experiment.add_tags(tags) print(f'In [add_tags]: Added these tags to the new experiment: {tags}') def set_name(self, name): self.experiment.set_name(name)
def log_hyperparameters_to_comet(clf, experiment): for i in range(len(clf.cv_results_["params"])): exp = Experiment( workspace="s0lvang", project_name="ideal-pancake-hyperparameter", api_key=globals.flags.comet_api_key, ) exp.add_tag("hp_tuning") exp.add_tags(globals.comet_logger.get_tags()) for k, v in clf.cv_results_.items(): if k == "params": exp.log_parameters(v[i]) else: exp.log_metric(k, v[i]) exp.end() old_experiment = ExistingExperiment( api_key=globals.flags.comet_api_key, previous_experiment=experiment.get_key(), ) globals.comet_logger = old_experiment
def run_main_loop(args, train_estimator, predict_estimator): total_steps = 0 train_steps = math.ceil(args.train_examples / args._batch_size) eval_steps = math.ceil(args.eval_examples / args._batch_size) if args.use_comet: experiment = Experiment(api_key=comet_ml_api_key, project_name=comet_ml_project, workspace=comet_ml_workspace) experiment.log_parameters(vars(args)) experiment.add_tags(args.tag) experiment.set_name(model_name(args)) else: experiment = None prefetch_inception_model() with tf.gfile.Open(os.path.join(suffixed_folder(args, args.result_dir), "eval.txt"), "a") as eval_file: for epoch in range(0, args.epochs, args.predict_every): logger.info(f"Training epoch {epoch}") train_estimator.train(input_fn=train_input_fn, steps=train_steps * args.predict_every) total_steps += train_steps * args.predict_every if args.use_comet: experiment.set_step(epoch) # logger.info(f"Evaluate {epoch}") # evaluation = predict_estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) # logger.info(evaluation) # save_evaluation(args, eval_file, evaluation, epoch, total_steps) # if args.use_comet: # experiment.log_metrics(evaluation) logger.info(f"Generate predictions {epoch}") predictions = predict_estimator.predict(input_fn=predict_input_fn) logger.info(f"Save predictions") save_predictions(args, suffixed_folder(args, args.result_dir), eval_file, predictions, epoch, total_steps, experiment) logger.info(f"Completed {args.epochs} epochs")
def train(args): # So I don't frigging forget what caused working models save_args(args) if args["use_tf_debug"]: hooks = [tf_debug.LocalCLIDebugHook()] else: hooks = [] if args["use_comet"]: # Add the following code anywhere in your machine learning file experiment = Experiment(api_key="bRptcjkrwOuba29GcyiNaGDbj", project_name="macgraph", workspace="davidhughhenrymack") experiment.log_multiple_params(args) if len(args["tag"]) > 0: experiment.add_tags(args["tag"]) train_size = sum( 1 for _ in tf.python_io.tf_record_iterator(args["train_input_path"])) logger.info(f"Training on {train_size} records") # ---------------------------------------------------------------------------------- estimator = get_estimator(args) train_spec = tf.estimator.TrainSpec( input_fn=gen_input_fn(args, "train"), max_steps=args["train_max_steps"] * 1000 if args["train_max_steps"] is not None else None, hooks=hooks) eval_spec = tf.estimator.EvalSpec(input_fn=gen_input_fn(args, "eval"), throttle_secs=args["eval_every"]) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def log(self, experiment=None): ''' Export all logs in the Comet.ml environment. See https://www.comet.ml/ for more details ''' # Initialize Comet.ml experience (naming, tags) for automatic logging project_name = 'Optimization' if self.comet_optimize else 'Summary' experiment_name = '{} - {} '.format(self.model_name, str(self.batch_size)) + ('ES+' if self.train_after_es else '') experiment_tags = [ self.model_name, self.monitor_val ] + (['ES+'] if self.train_after_es else []) + (['Pre-train'] if self.pretraining else []) if experiment == None: experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name=project_name, log_code=False, auto_param_logging=False, auto_metric_logging=False) experiment.set_name(experiment_name) experiment.add_tags(experiment_tags) # Export hyperparameters experiment.log_parameters(self.dataloader_params) experiment.log_parameters(self.training_params) # Export metrics values experiment.log_metrics({'Average accuracy' : np.mean(self.test_score['accuracy']), 'Std accuracy' : np.std(self.test_score['accuracy'])}) # Export metrics graphs for each pilot (accuracy, loss, confusion matrix) [ experiment.log_figure(figure_name='Confusion matrix {}'.format(pilot_idx), figure=plot_cm(self.conf_matrices, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] [ experiment.log_figure(figure_name='Loss pilot {}'.format(pilot_idx), figure=plot_loss(self.histories[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] fig, ax = plt.subplots(figsize=(10,6)) plot_full_barchart(self.test_score, n_pilots=self.n_pilots, title=' {} ConvNet model'.format(self.model_name), fig=fig) experiment.log_figure(figure_name='Accuracy barchart', figure=fig) if self.train_after_es: [ experiment.log_figure(figure_name='Loss pilot {} (ES+)'.format(pilot_idx), figure=plot_loss(self.histories_es[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] # Export model weights for each pilot [ experiment.log_asset('{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] experiment.end()
class CometML: def __init__(self, api_key, project_name, workspace, debug=True, tags=None): self._exp = Experiment( api_key=api_key, project_name=project_name, workspace=workspace, disabled=debug, ) if not (self._exp.alive or debug): raise RuntimeError("Cannot connect to Comet ML") self._exp.disable_mp() if tags is not None: self._exp.add_tags(tags) @property def run_name(self): return self._exp.get_key() def args(self, arg_text): self._exp.log_parameter("cmd args", arg_text) def meta(self, params): self._exp.log_parameters(params) def log(self, name, value, step): self._exp.log_metric( name=name, value=value, step=step, )
class _Experiment(object): def __init__(self, name): config_data = read_file_in_dir('./', name + '.json') if config_data is None: raise Exception("Configuration file doesn't exist: ", name) self.name = config_data['experiment_name'] dataset_config = config_data['dataset'] data_percentage = dataset_config['data_percentage'] batch_size = dataset_config['batch_size'] num_workers = dataset_config['num_workers'] data_files = dataset_config['data_file_path'] experiment_config = config_data['experiment'] self.epochs = experiment_config['num_epochs'] learning_rate = experiment_config['learning_rate'] model_config = config_data['model'] self.is_vae = model_config['is_vae'] hidden_size = model_config['hidden_size'] embedding_size = model_config['embedding_size'] self.is_variational = model_config['is_variational'] self.is_conditional = model_config['is_conditional'] class_label = model_config['class_label'] generation_config = config_data['generation'] max_length = generation_config['max_length'] prediction_type = ("Stochastic", "Deterministic")[generation_config["deterministic"]] temperature = generation_config['temperature'] self.experiment_dir = os.path.join(ROOT_STATS_DIR, self.name) self.log_comet = config_data['comet']['log_comet'] if self.log_comet: self.experiment = Experiment( api_key="CaoCCUZVjE0gXKA3cbtMKSSKL", project_name="image-captioning-251b", workspace="keshav919", ) # Load Datasets tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') self.train_loader, self.val_loader, self.test_loader = getDataloaders( data_files, max_length, batch_size, num_workers, tokenizer, data_percentage, class_label=class_label) # Setup Experiment self.current_epoch = 0 self.training_losses = [] self.bleu1_t = [] # Geeling: log bleu scores self.bleu4_t = [] self.bleu1_v = [] # Keshav: log bleu scores self.bleu4_v = [] self.val_losses = [] self.best_loss = float('inf') self.best_model = None # Save your best model in this field and use this in test method. if config_data['generation']: tags = [self.name, self.is_variational, prediction_type] hyper_params = { "Epochs": self.epochs, "Batch Size": batch_size, "Learning Rate": learning_rate, "Hidden Size": hidden_size, "Embedding Size": embedding_size, "Max Length": max_length, "Temperature": temperature } if self.log_comet: self.experiment.add_tags(tags) self.experiment.log_parameters(hyper_params) # Initialize Model self.tokenizer = tokenizer self.vocab_size = tokenizer.vocab_size self.model = getModel(config_data, self.vocab_size) # TODO: need to add KL divergence self.criterion = torch.nn.CrossEntropyLoss() self.optimizer = optim.Adam(self.model.parameters(), lr=learning_rate) self.init_model() self.load_experiment() # Load Experiment Data if available def load_experiment(self): """ Loads the experiment data if exists to resume training from last saved checkpoint. """ os.makedirs(ROOT_STATS_DIR, exist_ok=True) # Since we use comet, all our metrics are logged there rather than these directories. # Create the dir just for test output os.makedirs(self.experiment_dir, exist_ok=True) def init_model(self): """ Gets GPU ready to use """ self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.model.to(self.device) self.criterion.to(self.device) def loss_function(self, raw_outputs, hypothesis, mu, logvar): ceLoss = self.criterion(raw_outputs, hypothesis) if self.is_variational: klLoss = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) klLoss /= len(raw_outputs) ceLoss += klLoss return ceLoss def run(self): start_epoch = self.current_epoch for epoch in range( start_epoch, self.epochs): # loop over the dataset multiple times start_time = datetime.now() self.current_epoch = epoch if self.is_vae: ########################## VAE ############################## train_loss, bleu1_scores_t, bleu4_scores_t = self.train_vae() if self.log_comet: self.experiment.log_metrics({'Train_Loss': train_loss}, epoch=epoch) self.experiment.log_metrics( {'Train_Metric/BLEU-1': bleu1_scores_t}, epoch=epoch) self.experiment.log_metrics( {'Train_Metric/BLEU-4': bleu4_scores_t}, epoch=epoch) val_loss, bleu1_scores_v, bleu4_scores_v = self.val_vae() if self.log_comet: self.experiment.log_metrics({'Val_Loss': val_loss}, epoch=epoch) self.experiment.log_metrics( {'Val_Metric/BLEU-1': bleu1_scores_v}, epoch=epoch) self.experiment.log_metrics( {'Val_Metric/BLEU-4': bleu4_scores_v}, epoch=epoch) else: ########################## BERT ############################## train_loss, train_accu = self.train_bert() if self.log_comet: self.experiment.log_metrics({'Train_Loss': train_loss}, epoch=epoch) self.experiment.log_metrics({'Train_Accu': train_accu}, epoch=epoch) val_loss, val_accu = self.val_bert() if self.log_comet: self.experiment.log_metrics({'Val_Loss': val_loss}, epoch=epoch) self.experiment.log_metrics({'Val_Accu': val_accu}, epoch=epoch) # Early stopping if val_loss < self.best_loss: self.best_loss = val_loss torch.save(self.model, './saved_models/{}'.format(self.name)) if self.is_vae: self.test_vae() else: self.test_bert() ########################## VAE ############################## def train_vae(self): self.model.train() training_loss = 0 bleu1_scores = 0.0 bleu4_scores = 0.0 print_iter = 150 if_counter = 0 for i, (prem, hyp, _, _, lab) in enumerate(self.train_loader): self.model.zero_grad() prem = prem.long().to(self.device) hyp = hyp.long().to(self.device) lab = lab.to(self.device) # Forward pass preds, raw_outputs, mu, logvar = self.model( prem, hyp, lab, self.device, is_teacher_forcing_on=True, skip_generation=i % print_iter != 0, is_conditional=self.is_conditional) # Calculate loss and perform backprop loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1), hyp[:, 1:], mu, logvar) loss.backward() self.optimizer.step() # Log the training loss training_loss += loss.item() # View deterministic predictions if i % print_iter == 0: if_counter += 1 # Get the sentence without the <start> and <end> and other tags clean_premise_text, _ = clean_caption(prem, self.tokenizer) clean_preds_text, _ = clean_caption(preds, self.tokenizer) clean_targets_text, _ = clean_caption(hyp, self.tokenizer) # Calculate bleu scores b1 = calculate_bleu(bleu1, clean_preds_text, clean_targets_text) b4 = calculate_bleu(bleu4, clean_preds_text, clean_targets_text) bleu1_scores += (b1 / len(clean_preds_text)) bleu4_scores += (b4 / len(clean_preds_text)) print(self.current_epoch, i, ": ------ TRAIN ------") print("------ Actual Premise ------") print(clean_premise_text[0]) print("------ Actual Hypothesis ------") print(clean_targets_text[0]) print("------ Predicted Hypothesis ------") print(clean_preds_text[0]) print() return training_loss / ( i + 1), bleu1_scores / if_counter, bleu4_scores / if_counter def val_vae(self): self.model.eval() val_loss = 0 bleu1_scores = 0.0 bleu4_scores = 0.0 print_iter = 150 if_counter = 0 with torch.no_grad(): for i, (prem, hyp, _, _, lab) in enumerate(self.val_loader): prem = prem.long().to(self.device) hyp = hyp.long().to(self.device) lab = lab.to(self.device) # Forward pass preds, raw_outputs, mu, logvar = self.model( prem, hyp, lab, self.device, is_teacher_forcing_on=True, skip_generation=i % print_iter != 0, is_conditional=self.is_conditional) # Calculate loss and perform backprop loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1), hyp[:, 1:], mu, logvar) # Log the training loss val_loss += loss.item() # View deterministic predictions if i % print_iter == 0: if_counter += 1 # Get the sentence without the <start> and <end> and other tags clean_premise_text, _ = clean_caption(prem, self.tokenizer) clean_preds_text, _ = clean_caption(preds, self.tokenizer) clean_targets_text, _ = clean_caption(hyp, self.tokenizer) # Calculate bleu scores b1 = calculate_bleu(bleu1, clean_preds_text, clean_targets_text) b4 = calculate_bleu(bleu4, clean_preds_text, clean_targets_text) bleu1_scores += (b1 / len(clean_preds_text)) bleu4_scores += (b4 / len(clean_preds_text)) print(self.current_epoch, i, ": ------ VALIDATION ------") print("------ Actual Premise ------") print(clean_premise_text[0]) print("------ Actual Hypothesis ------") print(clean_targets_text[0]) print("------ Predicted Hypothesis ------") print(clean_preds_text[0]) print() return val_loss / ( i + 1), bleu1_scores / if_counter, bleu4_scores / if_counter def test_vae(self): self.model = torch.load('./saved_models/{}'.format(self.name)) self.model.eval() test_loss = 0 bleu1_scores = 0.0 bleu4_scores = 0.0 print_iter = 150 predicted = [] premises = [] labels = [] language = [] with torch.no_grad(): for i, (prem, hyp, _, _, lab) in enumerate(self.test_loader): prem = prem.long().to(self.device) hyp = hyp.long().to(self.device) lab = lab.to(self.device) # Forward pass _, raw_outputs, mu, logvar = self.model( prem, hyp, lab, self.device, is_teacher_forcing_on=True, is_conditional=self.is_conditional) preds, _, _, _ = self.model(prem, hyp, lab, self.device, is_teacher_forcing_on=False, is_conditional=self.is_conditional) # Calculate loss and perform backprop loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1), hyp[:, 1:], mu, logvar) # Log the training loss test_loss += loss.item() # Get the sentence without the <start> and <end> and other tags clean_premise_text, clean_premise_joined = clean_caption( prem, self.tokenizer) clean_preds_text, clean_preds_joined = clean_caption( preds, self.tokenizer) clean_targets_text, _ = clean_caption(hyp, self.tokenizer) predicted = predicted + clean_preds_joined premises = premises + clean_premise_joined labels = labels + lab.tolist() language = language + ['en'] * len(clean_premise_joined) # Calculate bleu scores b1 = calculate_bleu(bleu1, clean_preds_text, clean_targets_text) b4 = calculate_bleu(bleu4, clean_preds_text, clean_targets_text) bleu1_scores += (b1 / len(clean_preds_text)) bleu4_scores += (b4 / len(clean_preds_text)) if i % print_iter == 0: print(i, ": ------ TEST ------") print("------ Actual Premise ------") print(clean_premise_text[0]) print("------ Actual Hypothesis ------") print(clean_targets_text[0]) print("------ Predicted Hypothesis ------") print(clean_preds_text[0]) print() bleu1_scores = bleu1_scores / (i + 1) bleu4_scores = bleu4_scores / (i + 1) test_loss = test_loss / (i + 1) result_str = "Test Performance: Loss: {}, Bleu1: {}, Bleu4: {}".format( test_loss, bleu1_scores, bleu4_scores) self.log(result_str) if self.log_comet and self.is_vae: self.experiment.log_metrics({'Test_Loss': test_loss}) self.experiment.log_metrics({'Test_Metric/BLEU-1': bleu1_scores}) self.experiment.log_metrics({'Test_Metric/BLEU-4': bleu4_scores}) ds = { "premise": premises, "hypothesis": predicted, "label": labels, "lang_abv": language } df = pd.DataFrame( ds, columns=["premise", "hypothesis", "label", "lang_abv"]) df.to_csv("predicted_vae" + self.name + ".csv", index=False) return test_loss, bleu1_scores, bleu4_scores ########################## BERT ############################## def train_bert(self): self.model.train() training_loss = 0 print_iter = 50 total_pred = 0 correct_pred = 0 for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask, lab) in enumerate(self.train_loader): self.model.zero_grad() # Push data to GPU # (Model is pushed to GPU in line 127) prem_id = prem_id.long().to(self.device) hyp_id = hyp_id.long().to(self.device) prem_att_mask = prem_att_mask.long().to(self.device) hypo_att_mask = hypo_att_mask.long().to(self.device) lab = lab.long().to(self.device) # Prepare data as inputs to bert input_ids = torch.cat((prem_id, hyp_id), dim=1).long() attention_mask = torch.cat((prem_att_mask, hypo_att_mask), dim=1).long() label = lab.unsqueeze(1).long() # Forward pass outputs = self.model(input_ids, attention_mask=attention_mask, labels=label) # Calculate loss and perform backprop loss = outputs.loss loss.backward() self.optimizer.step() # Log the training loss training_loss += loss.item() # Get predicted labels predicted = torch.argmax(outputs.logits, 1) # calculate val accuracy = correct predictions/total predictions for j in range(len(lab)): total_pred += 1 if lab[j] == predicted[j]: correct_pred += 1 accu = correct_pred / total_pred # View deterministic predictions if i % print_iter == 0: print(self.current_epoch, i, ": ------ TRAIN ------") print("------ Actual Labels ------") print(lab) print("------ Predicted Labels ------") print(predicted) print("Current training accuracy: ", accu) # debugging code # if i==print_iter+1: # print("i is: ", i) # sys.exit() return training_loss / (i + 1), accu def val_bert(self): self.model.eval() val_loss = 0 print_iter = 50 total_pred = 0 correct_pred = 0 with torch.no_grad(): for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask, lab) in enumerate(self.val_loader): self.model.zero_grad() # Push data to GPU prem_id = prem_id.long().to(self.device) hyp_id = hyp_id.long().to(self.device) prem_att_mask = prem_att_mask.long().to(self.device) hypo_att_mask = hypo_att_mask.long().to(self.device) lab = lab.long().to(self.device) # Prepare data as inputs to bert input_ids = torch.cat((prem_id, hyp_id), dim=1) attention_mask = torch.cat((prem_att_mask, hypo_att_mask), dim=1) label = lab.unsqueeze(1) # Forward pass outputs = self.model(input_ids, attention_mask=attention_mask, labels=label) # Calculate loss and perform backprop loss = outputs.loss # Log the val loss val_loss += loss.item() # Get predicted labels predicted = torch.argmax(outputs.logits, 1) # calculate val accuracy = correct predictions/total predictions for j in range(len(lab)): total_pred += 1 if lab[j] == predicted[j]: correct_pred += 1 accu = correct_pred / total_pred # View deterministic predictions if i % print_iter == 0: print(self.current_epoch, i, ": ------ VAL ------") print("------ Actual Labels ------") print(lab) print("------ Predicted Labels ------") print(predicted) print("Current validation accuracy: ", accu) return val_loss / (i + 1), accu def test_bert(self): self.model.eval() test_loss = 0 print_iter = 50 total_pred = 0 correct_pred = 0 for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask, lab) in enumerate(self.test_loader): self.model.zero_grad() # Push data to GPU prem_id = prem_id.long().to(self.device) hyp_id = hyp_id.long().to(self.device) prem_att_mask = prem_att_mask.long().to(self.device) hypo_att_mask = hypo_att_mask.long().to(self.device) lab = lab.long().to(self.device) # Prepare data as inputs to bert input_ids = torch.cat((prem_id, hyp_id), dim=1) attention_mask = torch.cat((prem_att_mask, hypo_att_mask), dim=1) label = lab.unsqueeze(1) # Forward pass outputs = self.model(input_ids, attention_mask=attention_mask, labels=label) # Calculate loss and perform backprop loss = outputs.loss # Log the test loss test_loss += loss.item() # Get predicted labels predicted = torch.argmax(outputs.logits, 1) # calculate test accuracy = correct predictions/total predictions for j in range(len(lab)): total_pred += 1 if lab[j] == predicted[j]: correct_pred += 1 accu = correct_pred / total_pred # View deterministic predictions if i % print_iter == 0: print(self.current_epoch, i, ": ------ TEST ------") print("------ Actual Labels ------") print(lab) print("------ Predicted Labels ------") print(predicted) print("Current test accuracy: ", accu) if self.log_comet and (not self.is_vae): self.experiment.log_metrics({'Test_Loss': test_loss}) self.experiment.log_metrics({'Test_Accu': accu}) return test_loss / (i + 1), accu ########################## Log ############################## def save_model(self): root_model_path = os.path.join(self.experiment_dir, 'latest_model.pt') model_dict = self.model.state_dict() state_dict = { 'model': model_dict, 'optimizer': self.optimizer.state_dict() } torch.save(state_dict, root_model_path) def __record_stats(self, train_loss, bleu1_t, bleu4_t, val_loss, bleu1_v, bleu4_v): self.training_losses.append(train_loss) self.bleu1_t.append(bleu1_t) self.bleu4_t.append(bleu4_t) self.val_losses.append(val_loss) self.bleu1_v.append(bleu1_v) self.bleu4_v.append(bleu4_v) self.plot_stats() write_to_file_in_dir(self.experiment_dir, 'training_losses.txt', self.__training_losses) write_to_file_in_dir(self.experiment_dir, 'val_losses.txt', self.val_losses) write_to_file_in_dir(self.experiment_dir, 'bleu1.txt', self.bleu1) write_to_file_in_dir(self.experiment_dir, 'bleu4.txt', self.bleu4) def log(self, log_str, file_name=None): print(log_str) log_to_file_in_dir(self.experiment_dir, 'all.log', log_str) if file_name is not None: log_to_file_in_dir(self.experiment_dir, file_name, log_str) def log_epoch_stats(self, start_time): time_elapsed = datetime.now() - start_time time_to_completion = time_elapsed * (self.epochs - self.current_epoch - 1) train_loss = self.training_losses[self.current_epoch] val_loss = self.val_losses[self.current_epoch] summary_str = "Epoch: {}, Train Loss: {}, Val Loss: {}, Took {}, ETA: {}\n" summary_str = summary_str.format(self.current_epoch + 1, train_loss, val_loss, str(time_elapsed), str(time_to_completion)) self.log(summary_str, 'epoch.log')
random_state=42) checkpoint_callback = skopt.callbacks.CheckpointSaver( f'D:\\FINKI\\8_dps\\Project\\MODELS\\skopt_checkpoints\\{EXPERIMENT_ID}.pkl' ) hyperparameters_optimizer.fit(X_train, y_train, callback=[checkpoint_callback]) skopt.dump(hyperparameters_optimizer, f'saved_models\\{EXPERIMENT_ID}.pkl') y_pred = hyperparameters_optimizer.best_estimator_.predict(X_test) for i in range(len(hyperparameters_optimizer.cv_results_['params'])): exp = Experiment( api_key='A8Lg71j9LtIrsv0deBA0DVGcR', project_name=ALGORITHM, workspace="8_dps", auto_output_logging='native', ) exp.set_name(f'{EXPERIMENT_ID}_{i+1}') exp.add_tags([ DS, SEGMENTS_LENGTH, ]) for k, v in hyperparameters_optimizer.cv_results_.items(): if k == "params": exp.log_parameters(dict(v[i])) else: exp.log_metric(k, v[i]) exp.end() #%%
class CorefSolver(): def __init__(self, args): self.args = args self.data_utils = data_utils(args) self.disable_comet = args.disable_comet self.model = self.make_model( src_vocab=self.data_utils.vocab_size, tgt_vocab=self.data_utils.vocab_size, N=args.num_layer, dropout=args.dropout, entity_encoder_type=args.entity_encoder_type) print(self.model) if self.args.train: self.outfile = open(self.args.logfile, 'w') self.model_dir = make_save_dir(args.model_dir) # self.logfile = os.path.join(args.logdir, args.exp_name) # self.log = SummaryWriter(self.logfile) self.w_valid_file = args.w_valid_file def make_model(self, src_vocab, tgt_vocab, N=6, dropout=0.1, d_model=512, entity_encoder_type='linear', d_ff=2048, h=8): "Helper: Construct a model from hyperparameters." c = copy.deepcopy attn = MultiHeadedAttention(h, d_model) attn_ner = MultiHeadedAttention(1, d_model, dropout) ff = PositionwiseFeedForward(d_model, d_ff, dropout) position = PositionalEncoding(d_model, dropout) embed = Embeddings(d_model, src_vocab) word_embed = nn.Sequential(embed, c(position)) print('pgen', self.args.pointer_gen) if entity_encoder_type == 'transformer': # entity_encoder = nn.Sequential(embed, Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), 1)) print('transformer') entity_encoder = Seq_Entity_Encoder( embed, Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), 2)) elif entity_encoder_type == 'albert': albert_tokenizer = AlbertTokenizer.from_pretrained( 'albert-base-v2') albert = AlbertModel.from_pretrained('albert-base-v2') entity_encoder = Albert_Encoder(albert, albert_tokenizer, d_model) elif entity_encoder_type == 'gru': entity_encoder = RNNEncoder(embed, 'GRU', d_model, d_model, num_layers=1, dropout=0.1, bidirectional=True) print('gru') elif entity_encoder_type == 'lstm': entity_encoder = RNNEncoder(embed, 'LSTM', d_model, d_model, num_layers=1, dropout=0.1, bidirectional=True) print('lstm') if self.args.ner_at_embedding: model = EncoderDecoderOrg( Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N), DecoderOrg( DecoderLayerOrg(d_model, c(attn), c(attn), c(ff), dropout), N, d_model, tgt_vocab, self.args.pointer_gen), word_embed, word_embed, entity_encoder) else: if self.args.ner_last: decoder = Decoder( DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N, d_model, tgt_vocab, self.args.pointer_gen, self.args.ner_last) else: decoder = Decoder( DecoderLayer_ner(d_model, c(attn), c(attn), attn_ner, c(ff), dropout, self.args.fusion), N, d_model, tgt_vocab, self.args.pointer_gen, self.args.ner_last) model = EncoderDecoder( Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N), decoder, word_embed, word_embed, entity_encoder) # This was important from their code. # Initialize parameters with Glorot / fan_avg. for p in model.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) # levels = 3 # num_chans = [d_model] * (args.levels) # k_size = 5 # tcn = TCN(embed, d_model, num_channels, k_size, dropout=dropout) return model.cuda() def train(self): if not self.disable_comet: # logging hyper_params = { "num_layer": self.args.num_layer, "pointer_gen": self.args.pointer_gen, "ner_last": self.args.ner_last, "entity_encoder_type": self.args.entity_encoder_type, "fusion": self.args.fusion, "dropout": self.args.dropout, } COMET_PROJECT_NAME = 'summarization' COMET_WORKSPACE = 'timchen0618' self.exp = Experiment( api_key='mVpNOXSjW7eU0tENyeYiWZKsl', project_name=COMET_PROJECT_NAME, workspace=COMET_WORKSPACE, auto_output_logging='simple', auto_metric_logging=None, display_summary=False, ) self.exp.log_parameters(hyper_params) self.exp.add_tags([ '%s entity_encoder' % self.args.entity_encoder_type, self.args.fusion ]) if self.args.ner_last: self.exp.add_tag('ner_last') if self.args.ner_at_embedding: self.exp.add_tag('ner_at_embedding') self.exp.set_name(self.args.exp_name) self.exp.add_tag('coreference') print('ner_last ', self.args.ner_last) print('ner_at_embedding', self.args.ner_at_embedding) # dataloader & optimizer data_yielder = self.data_utils.data_yielder(num_epoch=100) optim = torch.optim.Adam(self.model.parameters(), lr=1e-7, betas=(0.9, 0.998), eps=1e-8, amsgrad=True) #get_std_opt(self.model) # entity_optim = torch.optim.Adam(self.entity_encoder.parameters(), lr=1e-7, betas=(0.9, 0.998), eps=1e-8, amsgrad=True) total_loss = [] start = time.time() print('*' * 50) print('Start Training...') print('*' * 50) start_step = 0 # if loading from checkpoint if self.args.load_model: state_dict = torch.load(self.args.load_model)['state_dict'] self.model.load_state_dict(state_dict) print("Loading model from " + self.args.load_model + "...") # encoder_state_dict = torch.load(self.args.entity_encoder)['state_dict'] # self.entity_encoder.load_state_dict(encoder_state_dict) # print("Loading entity_encoder from %s" + self.args.entity_encoder + "...") start_step = int(torch.load(self.args.load_model)['step']) print('Resume training from step %d ...' % start_step) warmup_steps = 10000 d_model = 512 lr = 1e-7 for step in range(start_step, self.args.total_steps): self.model.train() batch = data_yielder.__next__() optim.zero_grad() # entity_optim.zero_grad() #update lr if step % 400 == 1: lr = (1 / (d_model**0.5)) * min( (1 / (step / 4)**0.5), step * (1 / (warmup_steps**1.5))) for param_group in optim.param_groups: param_group['lr'] = lr # for param_group in entity_optim.param_groups: # param_group['lr'] = lr batch['src'] = batch['src'].long() batch['tgt'] = batch['tgt'].long() batch['ner'] = batch['ner'].long() batch['src_extended'] = batch['src_extended'].long() # forward the model if self.args.entity_encoder_type == 'albert': d = self.model.entity_encoder.tokenizer.batch_encode_plus( batch['ner_text'], return_attention_masks=True, max_length=10, add_special_tokens=False, pad_to_max_length=True, return_tensors='pt') ner_mask = d['attention_mask'].cuda().unsqueeze(1) ner = d['input_ids'].cuda() # print('ner', ner.size()) # print('ner_mask', ner_mask.size()) # print('src_mask', batch['src_mask'].size()) if self.args.entity_encoder_type == 'gru' or self.args.entity_encoder_type == 'lstm': ner_feat = self.model.entity_encoder( batch['ner'].transpose(0, 1), batch['cluster_len'])[1] elif self.args.entity_encoder_type == 'transformer': mask = gen_mask(batch['cluster_len']) ner_feat = self.model.entity_encoder(batch['ner'], mask) ner, ner_mask = self.data_utils.pad_ner_feature( ner_feat.squeeze(), batch['num_clusters'], batch['src'].size(0)) # print('ner', ner.size()) # print('ner_mask', ner_mask.size()) if self.args.ner_at_embedding: out = self.model.forward(batch['src'], batch['tgt'], ner, batch['src_mask'], batch['tgt_mask'], batch['src_extended'], len(batch['oov_list'])) else: out = self.model.forward(batch['src'], batch['tgt'], ner, batch['src_mask'], batch['tgt_mask'], batch['src_extended'], len(batch['oov_list']), ner_mask) # print out info pred = out.topk(1, dim=-1)[1].squeeze().detach().cpu().numpy()[0] gg = batch['src_extended'].long().detach().cpu().numpy()[0][:100] tt = batch['tgt'].long().detach().cpu().numpy()[0] yy = batch['y'].long().detach().cpu().numpy()[0] #compute loss & update loss = self.model.loss_compute(out, batch['y'].long()) loss.backward() optim.step() # entity_optim.step() total_loss.append(loss.detach().cpu().numpy()) # logging information if step % self.args.print_every_steps == 1: elapsed = time.time() - start print("Epoch Step: %d Loss: %f Time: %f lr: %6.6f" % (step, np.mean(total_loss), elapsed, optim.param_groups[0]['lr'])) self.outfile.write("Epoch Step: %d Loss: %f Time: %f\n" % (step, np.mean(total_loss), elapsed)) print( 'src:\n', self.data_utils.id2sent(gg, False, False, batch['oov_list'])) print( 'tgt:\n', self.data_utils.id2sent(yy, False, False, batch['oov_list'])) print( 'pred:\n', self.data_utils.id2sent(pred, False, False, batch['oov_list'])) print('oov_list:\n', batch['oov_list']) if ner_mask != None and not self.args.ner_at_embedding: pp = self.model.greedy_decode( batch['src_extended'].long()[:1], ner[:1], batch['src_mask'][:1], 100, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size, True, ner_mask[:1]) else: pp = self.model.greedy_decode( batch['src_extended'].long()[:1], ner[:1], batch['src_mask'][:1], 100, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size, True) pp = pp.detach().cpu().numpy() print( 'pred_greedy:\n', self.data_utils.id2sent(pp[0], False, False, batch['oov_list'])) print() start = time.time() if not self.disable_comet: # self.log.add_scalar('Loss/train', np.mean(total_loss), step) self.exp.log_metric('Train Loss', np.mean(total_loss), step=step) self.exp.log_metric('Learning Rate', optim.param_groups[0]['lr'], step=step) self.exp.log_text('Src: ' + self.data_utils.id2sent( gg, False, False, batch['oov_list'])) self.exp.log_text('Tgt:' + self.data_utils.id2sent( yy, False, False, batch['oov_list'])) self.exp.log_text('Pred:' + self.data_utils.id2sent( pred, False, False, batch['oov_list'])) self.exp.log_text('Pred Greedy:' + self.data_utils.id2sent( pp[0], False, False, batch['oov_list'])) self.exp.log_text('OOV:' + ' '.join(batch['oov_list'])) total_loss = [] ########################## # validation ########################## if step % self.args.valid_every_steps == 2: print('*' * 50) print('Start Validation...') print('*' * 50) self.model.eval() val_yielder = self.data_utils.data_yielder(1, valid=True) total_loss = [] fw = open(self.w_valid_file, 'w') for batch in val_yielder: with torch.no_grad(): batch['src'] = batch['src'].long() batch['tgt'] = batch['tgt'].long() batch['ner'] = batch['ner'].long() batch['src_extended'] = batch['src_extended'].long() ### ner ###### if self.args.entity_encoder_type == 'albert': d = self.model.entity_encoder.tokenizer.batch_encode_plus( batch['ner_text'], return_attention_masks=True, max_length=10, add_special_tokens=False, pad_to_max_length=True, return_tensors='pt') ner_mask = d['attention_mask'].cuda().unsqueeze(1) ner = d['input_ids'].cuda() if self.args.entity_encoder_type == 'gru' or self.args.entity_encoder_type == 'lstm': ner_feat = self.model.entity_encoder( batch['ner'].transpose(0, 1), batch['cluster_len'])[1] elif self.args.entity_encoder_type == 'transformer': mask = gen_mask(batch['cluster_len']) ner_feat = self.model.entity_encoder( batch['ner'], mask) ner, ner_mask = self.data_utils.pad_ner_feature( ner_feat.squeeze(), batch['num_clusters'], batch['src'].size(0)) ### ner ###### if self.args.ner_at_embedding: out = self.model.forward(batch['src'], batch['tgt'], ner, batch['src_mask'], batch['tgt_mask'], batch['src_extended'], len(batch['oov_list'])) else: out = self.model.forward(batch['src'], batch['tgt'], ner, batch['src_mask'], batch['tgt_mask'], batch['src_extended'], len(batch['oov_list']), ner_mask) loss = self.model.loss_compute(out, batch['y'].long()) total_loss.append(loss.item()) if self.args.ner_at_embedding: pred = self.model.greedy_decode( batch['src_extended'].long(), ner, batch['src_mask'], self.args.max_len, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size) else: pred = self.model.greedy_decode( batch['src_extended'].long(), ner, batch['src_mask'], self.args.max_len, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size, ner_mask=ner_mask) for l in pred: sentence = self.data_utils.id2sent( l[1:], True, self.args.beam_size != 1, batch['oov_list']) fw.write(sentence) fw.write("\n") fw.close() # files_rouge = FilesRouge() # scores = files_rouge.get_scores(self.w_valid_file, self.args.valid_tgt_file, avg=True) scores = cal_rouge_score(self.w_valid_file, self.args.valid_ref_file) r1_score = scores['rouge1'] r2_score = scores['rouge2'] print('=============================================') print('Validation Result -> Loss : %6.6f' % (sum(total_loss) / len(total_loss))) print(scores) print('=============================================') self.outfile.write( '=============================================\n') self.outfile.write('Validation Result -> Loss : %6.6f\n' % (sum(total_loss) / len(total_loss))) self.outfile.write( '=============================================\n') # self.model.train() # self.log.add_scalar('Loss/valid', sum(total_loss)/len(total_loss), step) # self.log.add_scalar('Score/valid', r1_score, step) if not self.disable_comet: self.exp.log_metric('Valid Loss', sum(total_loss) / len(total_loss), step=step) self.exp.log_metric('R1 Score', r1_score, step=step) self.exp.log_metric('R2 Score', r2_score, step=step) #Saving Checkpoint w_step = int(step / 10000) print('Saving ' + str(w_step) + 'w_model.pth!\n') self.outfile.write('Saving ' + str(w_step) + 'w_model.pth\n') model_name = str(w_step) + 'w_' + '%6.6f' % ( sum(total_loss) / len(total_loss) ) + '%2.3f_' % r1_score + '%2.3f_' % r2_score + 'model.pth' state = {'step': step, 'state_dict': self.model.state_dict()} torch.save(state, os.path.join(self.model_dir, model_name)) # entity_encoder_name = str(w_step) + '0w_' + '%6.6f'%(sum(total_loss)/len(total_loss)) + '%2.3f_'%r1_score + 'entity_encoder.pth' # state = {'step': step, 'state_dict': self.entity_encoder.state_dict()} # torch.save(state, os.path.join(self.model_dir, entity_encoder_name)) def test(self): #prepare model path = self.args.load_model # entity_encoder_path = self.args.entity_encoder state_dict = torch.load(path)['state_dict'] max_len = self.args.max_len model = self.model model.load_state_dict(state_dict) # entity_encoder_dict = torch.load(entity_encoder_path)['state_dict'] # self.entity_encoder.load_state_dict(entity_encoder_dict) pred_dir = make_save_dir(self.args.pred_dir) filename = self.args.filename #start decoding data_yielder = self.data_utils.data_yielder(num_epoch=1) total_loss = [] start = time.time() #file f = open(os.path.join(pred_dir, filename), 'w') self.model.eval() # decode_strategy = BeamSearch( # self.beam_size, # batch_size=batch.batch_size, # pad=self._tgt_pad_idx, # bos=self._tgt_bos_idx, # eos=self._tgt_eos_idx, # n_best=self.n_best, # global_scorer=self.global_scorer, # min_length=self.min_length, max_length=self.max_length, # return_attention=attn_debug or self.replace_unk, # block_ngram_repeat=self.block_ngram_repeat, # exclusion_tokens=self._exclusion_idxs, # stepwise_penalty=self.stepwise_penalty, # ratio=self.ratio) step = 0 for batch in data_yielder: #print(batch['src'].data.size()) step += 1 if step % 100 == 0: print('%d batch processed. Time elapsed: %f min.' % (step, (time.time() - start) / 60.0)) start = time.time() ### ner ### if self.args.entity_encoder_type == 'albert': d = self.model.entity_encoder.tokenizer.batch_encode_plus( batch['ner_text'], return_attention_masks=True, max_length=10, add_special_tokens=False, pad_to_max_length=True, return_tensors='pt') ner_mask = d['attention_mask'].cuda().unsqueeze(1) ner = d['input_ids'].cuda() else: ner_mask = None ner = batch['ner'].long() with torch.no_grad(): if self.args.beam_size == 1: if self.args.ner_at_embedding: out = self.model.greedy_decode( batch['src_extended'].long(), self.model.entity_encoder(ner), batch['src_mask'], max_len, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size) else: out = self.model.greedy_decode( batch['src_extended'].long(), self.model.entity_encoder(ner), batch['src_mask'], max_len, self.data_utils.bos, len(batch['oov_list']), self.data_utils.vocab_size, ner_mask=ner_mask) else: ret = self.beam_decode(batch, max_len, len(batch['oov_list'])) out = ret['predictions'] for l in out: sentence = self.data_utils.id2sent(l[1:], True, self.args.beam_size != 1, batch['oov_list']) #print(l[1:]) f.write(sentence) f.write("\n") def beam_decode(self, batch, max_len, oov_nums): src = batch['src'].long() src_mask = batch['src_mask'] src_extended = batch['src_extended'].long() bos_token = self.data_utils.bos beam_size = self.args.beam_size vocab_size = self.data_utils.vocab_size batch_size = src.size(0) def rvar(a): return a.repeat(beam_size, 1, 1) def rvar2(a): return a.repeat(beam_size, 1) def bottle(m): return m.view(batch_size * beam_size, -1) def unbottle(m): return m.view(beam_size, batch_size, -1) ### ner ### if self.args.entity_encoder_type == 'albert': d = self.model.entity_encoder.tokenizer.batch_encode_plus( batch['ner_text'], return_attention_masks=True, max_length=10, add_special_tokens=False, pad_to_max_length=True, return_tensors='pt') ner_mask = d['attention_mask'].cuda().unsqueeze(1) ner = d['input_ids'].cuda() else: ner_mask = None ner = batch['ner'].long() ner = self.model.entity_encoder(ner) if self.args.ner_at_embedding: memory = self.model.encode(src, src_mask, ner) else: memory = self.model.encode(src, src_mask) assert batch_size == 1 beam = [ Beam(beam_size, self.data_utils.pad, bos_token, self.data_utils.eos, min_length=self.args.min_length) for i in range(batch_size) ] memory = rvar(memory) ner = rvar(ner) src_mask = rvar(src_mask) src_extended = rvar2(src_extended) for i in range(self.args.max_len): if all((b.done() for b in beam)): break # Construct batch x beam_size nxt words. # Get all the pending current beam words and arrange for forward. inp = torch.stack([b.get_current_state() for b in beam]).t().contiguous().view(-1, 1) #inp -> [1, 3] inp_mask = inp < self.data_utils.vocab_size inp = inp * inp_mask.long() decoder_input = inp if self.args.ner_at_embedding: final_dist = self.model.decode(memory, ner, src_mask, decoder_input, None, src_extended, oov_nums) else: final_dist = self.model.decode(memory, ner, src_mask, decoder_input, None, src_extended, oov_nums, ner_mask=ner_mask) # final_dist, decoder_hidden, attn_dist_p, p_gen = self.seq2seq_model.model_copy.decoder( # decoder_input, decoder_hidden, # post_encoder_outputs, post_enc_padding_mask, # extra_zeros, post_enc_batch_extend_vocab # ) # # Run one step. # print('inp', inp.size()) # decoder_outputs: beam x rnn_size # (b) Compute a vector of batch*beam word scores. out = unbottle(final_dist) out[:, :, 2] = 0 #no unk # out.size -> [3, 1, vocab] # (c) Advance each beam. for j, b in enumerate(beam): b.advance(out[:, j]) # decoder_hidden = self.beam_update(j, b.get_current_origin(), beam_size, decoder_hidden) # (4) Extract sentences from beam. ret = self._from_beam(beam) return ret def _from_beam(self, beam): ret = {"predictions": [], "scores": []} for b in beam: n_best = self.args.n_best scores, ks = b.sort_finished(minimum=n_best) hyps = [] for i, (times, k) in enumerate(ks[:n_best]): hyp = b.get_hyp(times, k) hyps.append(hyp) ret["predictions"].append(hyps) ret["scores"].append(scores) return ret
class Logger: def __init__(self, send_logs, tags, parameters, experiment=None): self.stations = 5 self.send_logs = send_logs if self.send_logs: if experiment is None: json_loc = glob.glob("./**/comet_token.json")[0] with open(json_loc, "r") as f: kwargs = json.load(f) self.experiment = Experiment(**kwargs) else: self.experiment = experiment self.sent_mb = 0 self.speed_window = deque(maxlen=100) self.step_time = None self.current_speed = 0 if self.send_logs: if tags is not None: self.experiment.add_tags(tags) if parameters is not None: self.experiment.log_parameters(parameters) def begin_logging(self, episode_count, steps_per_ep, sigma, theta, step_time): self.step_time = step_time if self.send_logs: self.experiment.log_parameter("Episode count", episode_count) self.experiment.log_parameter("Steps per episode", steps_per_ep) self.experiment.log_parameter("theta", theta) self.experiment.log_parameter("sigma", sigma) def log_round(self, states, reward, cumulative_reward, info, loss, observations, step): self.experiment.log_histogram_3d(states, name="Observations", step=step) info = [[j for j in i.split("|")] for i in info] info = np.mean(np.array(info, dtype=np.float32), axis=0) try: # round_mb = np.mean([float(i.split("|")[0]) for i in info]) round_mb = info[0] except Exception as e: print(info) print(reward) raise e self.speed_window.append(round_mb) self.current_speed = np.mean(np.asarray(self.speed_window)/self.step_time) self.sent_mb += round_mb # CW = np.mean([float(i.split("|")[1]) for i in info]) CW = info[1] # stations = np.mean([float(i.split("|")[2]) for i in info]) self.stations = info[2] fairness = info[3] if self.send_logs: self.experiment.log_metric("Round reward", np.mean(reward), step=step) self.experiment.log_metric("Per-ep reward", np.mean(cumulative_reward), step=step) self.experiment.log_metric("Megabytes sent", self.sent_mb, step=step) self.experiment.log_metric("Round megabytes sent", round_mb, step=step) self.experiment.log_metric("Chosen CW", CW, step=step) self.experiment.log_metric("Station count", self.stations, step=step) self.experiment.log_metric("Current throughput", self.current_speed, step=step) self.experiment.log_metric("Fairness index", fairness, step=step) for i, obs in enumerate(observations): self.experiment.log_metric(f"Observation {i}", obs, step=step) self.experiment.log_metrics(loss, step=step) def log_episode(self, cumulative_reward, speed, step): if self.send_logs: self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step) self.experiment.log_metric("Speed", speed, step=step) self.sent_mb = 0 self.last_speed = speed self.speed_window = deque(maxlen=100) self.current_speed = 0 def end(self): if self.send_logs: self.experiment.end()
class CometMLMonitor(MonitorBase): """ Send scalar data and the graph to https://www.comet.ml. Note: 1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack. 2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze. Therefore the feature is disabled by default. """ def __init__(self, experiment=None, tags=None, **kwargs): """ Args: experiment (comet_ml.Experiment): if provided, invalidate all other arguments tags (list[str]): experiment tags kwargs: arguments used to initialize :class:`comet_ml.Experiment`, such as project name, API key, etc. Refer to its documentation for details. """ if experiment is not None: self._exp = experiment assert tags is None and len(kwargs) == 0 else: from comet_ml import Experiment kwargs.setdefault( 'log_code', True ) # though it's not functioning, git patch logging requires it kwargs.setdefault('auto_output_logging', None) self._exp = Experiment(**kwargs) if tags is not None: self._exp.add_tags(tags) self._exp.set_code("Code logging is impossible ...") self._exp.log_dependency('tensorpack', __git_version__) @property def experiment(self): """ The :class:`comet_ml.Experiment` instance. """ return self._exp def _before_train(self): self._exp.set_model_graph(tf.get_default_graph()) @HIDE_DOC def process_scalar(self, name, val): self._exp.log_metric(name, val, step=self.global_step) @HIDE_DOC def process_image(self, name, val): self._exp.set_step(self.global_step) for idx, v in enumerate(val): log_name = "{}_step{}{}".format( name, self.global_step, "_" + str(idx) if len(val) > 1 else "") self._exp.log_image(v, image_format="jpeg", name=log_name, image_minmax=(0, 255)) def _after_train(self): self._exp.end() def _after_epoch(self): self._exp.log_epoch_end(self.epoch_num)
def main( logger, optimized_function, optimizer_config_file, model_config_file, project_name, work_space, tags, init_psi, n_samples_per_dim, ): model_config = getattr(__import__(model_config_file), 'model_config') model_config["num_repetitions"] = n_samples_per_dim optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') init_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',')]).float().to(device) psi_dim = len(init_psi) optimized_function_cls = str_to_class(optimized_function) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) experiment.log_parameters( {"model_{}".format(key): value for key, value in model_config.items()}) logger = str_to_class(logger)(experiment) y_model = optimized_function_cls(device=device, psi_init=init_psi) model = VoidModel(y_model=y_model, psi=init_psi, **model_config) optimizer = VoidOptimizer(oracle=model, x=init_psi, logger=logger, n_samples=model_config["K"], **optimizer_config) current_psi, status, history = optimizer.optimize() try: logger.log_optimizer(optimizer) logger.log_grads(model, y_sampler=y_model, current_psi=x_k, num_repetitions=30000, n_samples=100, log_grad_diff=True) logger.log_performance(y_sampler=y_model, current_psi=current_psi, n_samples=5000) except Exception as e: print(e) raise
def comet_lgbm(save_path): from comet_ml import Experiment exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj", project_name="baseline", workspace="gdreiman1") exp.log_code = True import pickle import pandas as pd import lightgbm as lgb import numpy as np import sklearn import matplotlib.pyplot as plt from sklearn.metrics import precision_recall_fscore_support as prf #%% def single_roc(y_preds,y_true): from sklearn.metrics import roc_curve, auc,precision_recall_curve fpr, tpr, _ = roc_curve(y_true, y_preds) roc_auc = auc(fpr, tpr) plt.figure() lw = 2 plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') precision, recall, thresholds = precision_recall_curve(y_true, y_preds) plt.plot(recall, precision, color='blue', lw=lw, label='Precision vs Recall') # show the plot plt.legend(loc="lower right") plt.show() def multi_roc(y_preds,y_true,name,n_classes): import collections nested_dict = lambda: collections.defaultdict(nested_dict) data_store = nested_dict() from sklearn.metrics import roc_curve, auc from scipy import interp from itertools import cycle lw = 2 name_store = ['Active', 'Inactive', 'Inconclusive'] fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_preds[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr["micro"], tpr["micro"], _ = roc_curve(y_true[:, i].ravel(), y_preds[:, i].ravel()) roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) # Compute macro-average ROC curve and ROC area # First aggregate all false positive rates all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)])) # Then interpolate all ROC curves at this points mean_tpr = np.zeros_like(all_fpr) for i in range(n_classes): mean_tpr += interp(all_fpr, fpr[i], tpr[i]) # Finally average it and compute AUC mean_tpr /= n_classes fpr["macro"] = all_fpr tpr["macro"] = mean_tpr roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) # Plot all ROC curves plt.figure() plt.plot(fpr["micro"], tpr["micro"], label='micro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["micro"]), color='deeppink', linestyle=':', linewidth=4) plt.plot(fpr["macro"], tpr["macro"], label='macro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["macro"]), color='navy', linestyle=':', linewidth=4) colors = cycle(['aqua', 'darkorange', 'cornflowerblue','green']) for i, color in zip(range(n_classes), colors): plt.plot(fpr[i], tpr[i], color=color, lw=lw, label='ROC curve of '+ name_store[i]+'(area = {1:0.2f})' ''.format(i, roc_auc[i])) plt.plot([0, 1], [0, 1], 'k--', lw=lw) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') #plt.title('Multi-class ROC for '+name+' Split= '+str(count+1)) plt.title('Multi-class ROC for '+name) plt.legend(loc="lower right") #plt.show() #%% #save_path = r'C:\Users\gdrei\Dropbox\UCL\Thesis\May_13\AID_1345083_processed.pkl' model_type = 'lgbm' #get data cleaned pickle_off = open(save_path,'rb') activity_table=pickle.load(pickle_off) pickle_off.close() #get length of MFP fp_length = len(activity_table.iloc[5]['MFP']) from sklearn.preprocessing import StandardScaler, LabelEncoder scaler = StandardScaler(copy = False) le = LabelEncoder() labels = le.fit_transform(activity_table['PUBCHEM_ACTIVITY_OUTCOME']) #split data: from sklearn.model_selection import StratifiedShuffleSplit splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.5, train_size=None, random_state=2562) X_mfp = np.concatenate(np.array(activity_table['MFP'])).ravel() X_mfp = X_mfp.reshape((-1,fp_length)) for train_ind, test_ind in splitter.split(X_mfp,labels): # standardize data X_train_molchars_std = scaler.fit_transform(np.array(activity_table.iloc[train_ind,4:])) X_test_molchars_std = scaler.transform(np.array(activity_table.iloc[test_ind,4:])) X_train = np.concatenate((X_mfp[train_ind,:],X_train_molchars_std),axis = 1) X_test = np.concatenate((X_mfp[test_ind,:],X_test_molchars_std),axis = 1) y_train = labels[train_ind] y_test = labels[test_ind] #X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X,labels,test_size = .5, shuffle = True, stratify = labels, random_state = 2562) bin_y_train, bin_y_test = [1 if x ==2 else x for x in y_train],[1 if x ==2 else x for x in y_test] #do light gbm #need to make a lib svm file train_data = lgb.Dataset(X_train,label=y_train) test_data = lgb.Dataset(X_test,label=y_test) #make model class lgbm_model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=500, subsample_for_bin=200000, objective='binary', is_unbalance=True, min_split_gain=0.0, min_child_weight=0.001, min_child_samples=20, subsample=1.0, subsample_freq=0, colsample_bytree=1.0, reg_alpha=0.0, reg_lambda=0.0, random_state=None, n_jobs=-1, silent=True, importance_type='split') #train model trained_mod = lgbm_model.fit(X_train,y_train) #predict classes and class_probs test_class_preds = lgbm_model.predict(X_test) test_prob_preds = lgbm_model.predict_proba(X_test) #calculate Class report class_rep = sklearn.metrics.classification_report(y_test,test_class_preds) print(class_rep) if len(set(y_test)) == 2: single_roc(test_prob_preds[:,1],y_test) prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None) else: from tensorflow.keras.utils import to_categorical multi_roc(test_prob_preds,to_categorical(y_test),'',3) prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None) #%% '''Comet Saving Zone''' #get AID number import ntpath #get base file name folder,base = ntpath.split(save_path) #split file name at second _ assumes file save in AID_xxx_endinfo.pkl AID, _,end_info = base.rpartition('_') #save data location, AID info, and version info exp.log_dataset_info(name = AID, version = end_info, path = save_path) #save model params exp.log_parameters(trained_mod.get_params()) #save metrics report to comet if len(f_1) == 2: for i,name in enumerate(['Active','Inactive']): exp.log_metric('f1 class '+name, f_1[i]) exp.log_metric('Recall class'+name,rec[i]) exp.log_metric('Precision class'+name, prec[i]) else: for i,name in enumerate(['Active','Inconclusive','Inactive']): exp.log_metric('f1 class '+str(i), f_1[i]) exp.log_metric('Recall class'+str(i),rec[i]) exp.log_metric('Precision class'+str(i), prec[i]) #exp.log_metric('f1 class '+str(i), f_1[i]) #exp.log_metric('Recall class'+str(i),rec[i]) #exp.log_metric('Precision class'+str(i), prec[i]) exp.log_other('Classification Report',class_rep) #save model in data_folder with comet experiement number associated exp_num = exp.get_key() model_save = folder+'\\'+model_type+'_'+exp_num+'.pkl' pickle_on = open(model_save,'wb') pickle.dump(trained_mod,pickle_on) pickle_on.close() #log trained model location exp.log_other('Trained Model Path',model_save) #save some informatvie tags: tags = [AID,end_info,model_type] exp.add_tags(tags) #save ROC curve exp.log_figure(figure_name = 'ROC-Pres/Recall',figure=plt) plt.show() #tell comet that the experiement is over exp.end()
# comet.ml experiment information if args.comet_track: experiment = Experiment(api_key="", project_name="", workspace="", auto_output_logging="simple", log_git_metadata=False, log_git_patch=False) if experiment.alive is False: raise Exception("Could not connect to comet.ml!") # disable automatic logging # experiment.auto_param_logging=False # parameters all args above as parameters experiment.log_parameters(args.__dict__) experiment.add_tags([args.task, args.model]) else: # pseudoclass to enable "with experiment.train()" when not using comet experiment = voidExperiment() # GloVe has 4 valid embedding dimensions, assert that one of them is chosen if not args.no_glove: assert args.dim_emb in [50, 100, 200, 300], 'Choose valid GloVe dimension' print('\nEvaluating incremental outputs of task: {}, with model {}.\n'.format( args.task, args.model)) seq2seq=True if args.task in seq2label_tasks: assert args.model not in ['lstm_crf', 'bilstm_crf'], 'CRF cannot be used with seq2label task' seq2seq = False
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', type=str, default='gcn') parser.add_argument('--type', type=str, default='base') parser.add_argument('--runs', type=int, default=10) parser.add_argument('--epochs', type=int, default=200) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--hidden_dim', type=int, default=300) parser.add_argument('--num_layers', type=int, default=5) args = parser.parse_args() assert args.model in ['gcn', 'sage', 'gin'] assert args.type in [ 'base', 'transfer', 'transfer-damaged', 'self-transfer', 'self-transfer-damaged' ] assert args.runs >= 1 assert args.epochs >= 1 assert args.lr > 0 assert args.hidden_dim > 0 assert args.num_layers > 0 # --------------------------------------------------- # MODEL # --------------------------------------------------- model = networks[args.model](in_channels=dataset.num_features, hidden_channels=args.hidden_dim, out_channels=dataset.num_tasks, num_conv_layers=args.num_layers).to(device) # --------------------------------------------------- # EXPERIMENT DETAILS # --------------------------------------------------- print('Graph Classification Experiment') print('Mol_HIV task') print(exp_description[args.type]) print('----------------------------------------------') print('Model: {}'.format(args.model)) print('Number of runs: {}'.format(args.runs)) print('Number of epochs: {}'.format(args.epochs)) print('Learning rate: {}'.format(args.lr)) print() print(model) # --------------------------------------------------- # EXPERIMENT LOOP # --------------------------------------------------- for run in range(args.runs): print() print('Run #{}'.format(run + 1)) # Model initialisation if args.type == 'base': model.reset_parameters() elif args.type in ['transfer', 'transfer-damaged']: # Pretrain on Mol-BBBP model.reset_parameters() bbbp_optimiser = optim.Adam(model.parameters(), lr=0.001) to_damage = args.type == 'transfer-damaged' print('Pretraining model on Mol-BBBP...') best_val_acc = pretrain_molbbbp(model, device, bbbp_evaluator, bbbp_optimiser, args.model, damage=to_damage) print('Validation accuracy: {:.3}'.format(best_val_acc)) model.load_state_dict( torch.load('molbbbp_models/{}_molbbbp.pth'.format(args.model))) elif args.type in ['self-transfer', 'self-transfer-damaged']: # Pretrain on Mol-HIV Source Split model.reset_parameters() source_optimiser = optim.Adam(model.parameters(), lr=0.001) to_damage = args.type == 'self-transfer-damaged' print('Pretraining model on Mol-HIV Source Task...') best_val_acc = pretrain_source_molhiv(model, device, evaluator, source_optimiser, args.model, damage=to_damage) print('Validation accuracy: {:.3}'.format(best_val_acc)) model.load_state_dict( torch.load('molhiv/{}_source_molhiv.pth'.format(args.model))) # Comet Experiment experiment = Experiment(project_name='graph-classification', display_summary_level=0, auto_metric_logging=False) experiment.add_tags([args.model, args.type]) experiment.log_parameters({ 'hidden_dim': args.hidden_dim, 'num_features': dataset.num_features, 'num_classes': dataset.num_tasks, 'learning_rate': args.lr, 'num_epochs': args.epochs, }) # Mol-HIV Target Training print('Training on Mol-HIV') optimizer = optim.Adam(model.parameters(), args.lr) for epoch in tqdm(range(args.epochs)): train_loss = train(model, device, target_loader, optimizer) train_performance = eval(model, device, target_loader, evaluator) experiment.log_metric('train_loss', train_loss.item(), step=epoch) experiment.log_metric('train_roc-auc', train_performance[dataset.eval_metric], step=epoch) experiment.end()
'minibatch_size': minibatch_size, 'lr': learning_rate, 'discount_factor': discount_factor, 'random_process_theta': random_process_args['theta'], 'log_interval_steps': log_interval_steps, 'train_data_shape': train_data_df.shape, 'test_data_shape': test_data_df.shape, 'dataset_name': dataset_name, 'device_type': device_type } print('Running with params: %s' % str(params)) if log_comet: experiment.log_parameters(params) experiment.add_tags(comet_tags) if plot_stocks: experiment.log_image('train_stocks_plot.png', 'train_window_stocks') if test_stocks_plot_fig is not None: experiment.log_image('test_stocks_plot.png', 'test_window_stocks') num_stocks = train_data_df.shape[1] num_states_and_actions = num_stocks # init DDPG agent agent = DDPG(num_states_and_actions, num_states_and_actions, minibatch_size, random_process_args, learning_rate=learning_rate, discount_factor=discount_factor,
ap, f1_max, precision, recall, f1_max_th, fig_pre_rec, fig_th_pre_rec = precision_recall(y_test, score_test, limit=1000, label_anomaly=labels[0]) #fig_score_train = plot_score(score_train, 'train') #fig_score_val = plot_score(score_val, 'validation') fig_score_test = plot_score(score_test, 'test', 10, labels=y_test, th=f1_max_th) fig_cumsum = pca.plot_cumsum() y_pred, conf_matrix = predict(score_test, f1_max_th, y_test, labels) experiment.add_tags([data, metric]) parameters = {'var': var, 'pc': pca.pcs_, 'metric': metric} experiment.log_parameters(parameters) experiment.log_metric('ap', ap) experiment.log_metric('f1', f1_max) experiment.log_metric('precision', precision) experiment.log_metric('recall', recall) experiment.log_metric('train_time', pca.time_) experiment.log_parameter('th_f1', f1_max_th) experiment.log_figure('cumsum', fig_cumsum) experiment.log_figure('score_test',fig_score_test) experiment.log_figure('precision_recall',fig_pre_rec) experiment.log_figure('th_pre_rec_f1', fig_th_pre_rec) experiment.log_confusion_matrix(matrix=conf_matrix, labels=labels) experiment.end()
evaluation_iterator = pieces.validation_iterator or pieces.iterator evaluation_dataset = pieces.test_dataset else: raise ConfigurationError("Need to use am-trainer.") params.assert_empty('base train command') if args.comet is not None: experiment = Experiment(api_key=args.comet, workspace=args.workspace, project_name=args.project, parse_args=False, auto_output_logging=None) if args.tags: experiment.add_tags(args.tags) with open(args.param_path) as fil: code = "".join(fil.readlines()) code += "\n\n#=============Full details=============\n\n" code += _jsonnet.evaluate_file(args.param_path) code += "\n\n#=============IMPORTANT: overwritten options============\n\n" code += args.overrides experiment.set_code(code) code_data = json.loads(_jsonnet.evaluate_file(args.param_path)) experiment.log_parameter( "bert", "bert" in code_data["dataset_reader"]["token_indexers"]) experiment.log_parameter( "elmo", "elmo" in code_data["dataset_reader"]["token_indexers"]) experiment.log_parameter("model_directory", serialization_dir) experiment.log_parameter("cuda_device", cuda_device) experiment.log_parameter("corpora", code_data["iterator"]["formalisms"])