def train_epoch(self, iterator, info=None, num_steps=None, epoch_idx=0): """Runs one standard training pass over the training dataloader. By default, this method will iterate over the given iterator and call ``self.train_batch`` over each batch. If ``scheduler_step_freq`` is set, this default method will also step the scheduler accordingly. You do not need to call ``train_batch`` in this method if you plan to implement a custom optimization/training routine here. You may find ``ray.util.sgd.utils.AverageMeterCollection`` useful when overriding this method. See example below: .. code-block:: python def train_epoch(self, ...): meter_collection = AverageMeterCollection() self.model.train() for batch in iterator: # do some processing metrics = {"metric_1": 1, "metric_2": 3} # dict of metrics # This keeps track of all metrics across multiple batches meter_collection.update(metrics, n=len(batch)) # Returns stats of the meters. stats = meter_collection.summary() return stats Args: iterator (iter): Iterator over the training data for the entire epoch. This iterator is expected to be entirely consumed. info (Optional[dict]): Dictionary for information to be used for custom training operations. num_steps (Optional[int]): Number of steps in the iterator. epoch_idx (int): Index of current epoch. Returns: A dict of metrics from training. """ if not hasattr(self, "model"): raise RuntimeError("Either set self.model in setup function or " "override this method to implement a custom " "training loop.") info = info or {} info.update({ NUM_STEPS: num_steps, USE_FP16: self.use_fp16, "epoch_idx": epoch_idx }) model = self.model scheduler = None if hasattr(self, "scheduler"): scheduler = self.scheduler if self.use_tqdm and self.world_rank == 0: desc = "" if info is not None and "epoch_idx" in info: if "num_epochs" in info: desc = f"{info['epoch_idx'] + 1}/{info['num_epochs']}e" else: desc = f"{info['epoch_idx'] + 1}e" # TODO: Implement len for Dataset? total = info[NUM_STEPS] if total is None: if hasattr(iterator, "__len__"): total = len(iterator) _progress_bar = tqdm(total=total, desc=desc, unit="batch", leave=False) metric_meters = AverageMeterCollection() model.train() for batch_idx, batch in enumerate(iterator): batch_info = { "batch_idx": batch_idx, "global_step": self.global_step } batch_info.update(info) metrics = self.train_batch(batch, batch_info=batch_info) if self.use_tqdm and self.world_rank == 0: _progress_bar.n = batch_idx + 1 postfix = {} if "train_loss" in metrics: postfix.update(loss=metrics["train_loss"]) _progress_bar.set_postfix(postfix) if scheduler and self.scheduler_step_freq == SCHEDULER_STEP_BATCH: scheduler.step() metric_meters.update(metrics, n=metrics.pop(NUM_SAMPLES, 1)) self.global_step += 1 if scheduler and self.scheduler_step_freq == SCHEDULER_STEP_EPOCH: scheduler.step() return metric_meters.summary()
def train_epoch(self, iterator, info): model = self.get_model() # Enable train mode. self.model.train() # Enable gradients. torch.set_grad_enabled(True) if self.is_function_implemented("on_train_epoch_start", model): model.on_train_epoch_start() if self.use_tqdm and self.world_rank == 0: desc = "" if info is not None and "epoch_idx" in info: if "num_epochs" in info: desc = f"{info['epoch_idx'] + 1}/{info['num_epochs']}e" else: desc = f"{info['epoch_idx'] + 1}e" # TODO: Implement len for Dataset? total = info[NUM_STEPS] if total is None: if hasattr(iterator, "__len__"): total = len(iterator) _progress_bar = tqdm(total=total, desc=desc, unit="batch", leave=False) # Output for each batch. epoch_outputs = [] for batch_idx, batch in enumerate(iterator): batch_info = { "batch_idx": batch_idx, "global_step": self.global_step } batch_info.update(info) batch_output = self.train_batch(batch, batch_info=batch_info) # batch output for each optimizer. epoch_outputs.append(batch_output) should_stop = batch_output["signal"] == -1 if self.use_tqdm and self.world_rank == 0: _progress_bar.n = batch_idx + 1 postfix = {} if "training_loss" in batch_output: postfix.update(loss=batch_output["training_loss"]) _progress_bar.set_postfix(postfix) for s_dict, scheduler in zip(self.scheduler_dicts, self.schedulers): if s_dict["interval"] == SCHEDULER_STEP_BATCH: scheduler.step() self.global_step += 1 if should_stop: break processed_outputs = None if is_overridden("training_epoch_end", model): raw_outputs = [eo["raw_output"] for eo in epoch_outputs] processed_outputs = model.training_epoch_end(raw_outputs) if processed_outputs is not None: if isinstance(processed_outputs, torch.Tensor): return_output = {"train_loss": processed_outputs} elif isinstance(processed_outputs, Result): raise ValueError("Result objects are not supported. Please " "return a dictionary instead.") elif isinstance(processed_outputs, dict): return_output = processed_outputs else: raise TypeError("training_epoch_end returned an invalid " "type. It must return a Tensor, Result, " "or dict.") else: # User did not override training_epoch_end assert isinstance(epoch_outputs, list) # Use AverageMeterCollection util to reduce results. meter_collection = AverageMeterCollection() for o in epoch_outputs: num_samples = o.pop(NUM_SAMPLES, 1) raw_output = o["raw_output"] if isinstance(raw_output, dict): meter_collection.update(raw_output, num_samples) elif isinstance(raw_output, torch.Tensor): meter_collection.update({"train_loss": o["training_loss"]}, num_samples) return_output = meter_collection.summary() if self.is_function_implemented("on_train_epoch_end", model): model.on_train_epoch_end( [eo.get("raw_output") for eo in epoch_outputs]) for s_dict, scheduler in zip(self.scheduler_dicts, self.schedulers): if s_dict["interval"] == SCHEDULER_STEP_EPOCH: scheduler.step() return return_output
def train_epoch(self, iterator, info): """Runs one standard training pass over the training dataloader. By default, this method will iterate over the given iterator and call ``self.train_batch`` over each batch. If ``scheduler_step_freq`` is set, this default method will also step the scheduler accordingly. You do not need to call ``train_batch`` in this method if you plan to implement a custom optimization/training routine here. You may find ``ray.util.sgd.utils.AverageMeterCollection`` useful when overriding this method. See example below: .. code-block:: python def train_epoch(self, ...): meter_collection = AverageMeterCollection() self.model.train() for batch in iterator: # do some processing metrics = {"metric_1": 1, "metric_2": 3} # dict of metrics # This keeps track of all metrics across multiple batches meter_collection.update(metrics, n=len(batch)) # Returns stats of the meters. stats = meter_collection.summary() return stats Args: iterator (iter): Iterator over the training data for the entire epoch. This iterator is expected to be entirely consumed. info (dict): Dictionary for information to be used for custom training operations. Returns: A dict of metrics from training. """ if self.use_tqdm and self.world_rank == 0: desc = "" if info is not None and "epoch_idx" in info: if "num_epochs" in info: desc = "{}/{}e".format(info["epoch_idx"] + 1, info["num_epochs"]) else: desc = "{}e".format(info["epoch_idx"] + 1) _progress_bar = tqdm(total=info[NUM_STEPS] or len(self.train_loader), desc=desc, unit="batch", leave=False) metric_meters = AverageMeterCollection() self.model.train() for batch_idx, batch in enumerate(iterator): batch_info = { "batch_idx": batch_idx, "global_step": self.global_step } batch_info.update(info) metrics = self.train_batch(batch, batch_info=batch_info) if self.use_tqdm and self.world_rank == 0: _progress_bar.n = batch_idx + 1 postfix = {} if "train_loss" in metrics: postfix.update(loss=metrics["train_loss"]) _progress_bar.set_postfix(postfix) if self.scheduler and batch_info.get( SCHEDULER_STEP) == SCHEDULER_STEP_BATCH: self.scheduler.step() metric_meters.update(metrics, n=metrics.pop(NUM_SAMPLES, 1)) self.global_step += 1 if self.scheduler and info.get(SCHEDULER_STEP) == SCHEDULER_STEP_EPOCH: self.scheduler.step() return metric_meters.summary()
def validate(self, validation_loader, info): meter_collection = AverageMeterCollection() model = self.model n_layers = self.config["n_layers"] n_hidden = self.config["n_hidden"] n_heads = self.config["n_heads"] batch_size = self.config["batch_size"] num_workers = self.config["sampling_num_workers"] g = self.g train_nid = self.train_nid val_nid = self.val_nid test_nid = self.test_nid device = 0 model.eval() with torch.no_grad(): x = g.ndata["features"] for i, layer in enumerate(self.convs): if i < n_layers - 1: y = torch.zeros( g.number_of_nodes(), n_hidden * n_heads if i != len(self.convs) - 1 else self.n_classes) else: y = torch.zeros( g.number_of_nodes(), n_hidden if i != len(self.convs) - 1 else self.n_classes) sampler = dgl.dataloading.MultiLayerFullNeighborSampler(1) collator = NodeCollator(g, torch.arange(g.number_of_nodes()), sampler) dataloader = DataLoader(collator.dataset, collate_fn=collator.collate, batch_size=batch_size, shuffle=False, drop_last=False, num_workers=num_workers) for input_nodes, output_nodes, blocks in dataloader: block = blocks[0] # print("block:",block) block = block.int().to(device) h = x[input_nodes].to(device) h_dst = x[output_nodes].to(device) if i != len(self.convs) - 1: h = layer(block, (h, h_dst)).flatten(1) else: h = layer(block, (h, h_dst)).mean(1) h = h.log_softmax(dim=-1) y[output_nodes] = h.cpu() x = y pred = y labels = g.ndata["labels"] _, val_acc, test_acc = compute_acc(pred[train_nid], labels[ train_nid]), compute_acc(pred[val_nid], labels[val_nid]), \ compute_acc(pred[test_nid], labels[test_nid]) metrics = { "num_samples": pred.size(0), "val_acc": val_acc.item(), "test_acc": test_acc.item() } meter_collection.update(metrics, n=metrics.pop("num_samples", 1)) status = meter_collection.summary() return status