def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._per_slot_batch_size, self._global_batch_size = util.calculate_batch_sizes( self.get_hparams(), self.env.experiment_config.slots_per_trial(), "NoOpTrial", )
def __init__(self, *arg: Any, **kwarg: Any) -> None: det.TrialContext.__init__(self, *arg, **kwarg) estimator._EstimatorReducerContext.__init__(self, self.distributed.allgather) self._per_slot_batch_size, self._global_batch_size = util.calculate_batch_sizes( self.get_hparams(), self.env.experiment_config.slots_per_trial(), "EstimatorTrial", ) self.experimental = EstimatorExperimentalContext( self.env, self.distributed, self._per_slot_batch_size, ) if self.distributed.size > 1: optimizations_config = self.env.experiment_config.get_optimizations_config( ) self.aggregation_frequency = cast( int, optimizations_config.get("aggregation_frequency")) self.fp16_compression = cast( bool, optimizations_config.get("gradient_compression")) self.average_aggregated_gradients = cast( bool, optimizations_config.get("average_aggregated_gradients")) self.optimizer_initialized = False self.dataset_initialized = False
def __init__(self, *args: Any, **kwargs: Any) -> None: det.TrialContext.__init__(self, *args, **kwargs) pytorch._PyTorchReducerContext.__init__(self, self.distributed.allgather) self._per_slot_batch_size, self._global_batch_size = util.calculate_batch_sizes( self.get_hparams(), self.env.experiment_config.slots_per_trial(), "PyTorchTrial", ) self._distributed_backend = det._DistributedBackend() self.device = self._init_device() # Track which types we have issued warnings for in to_device(). self._to_device_warned_types = set() # type: Set[Type] # The following attributes are initialized during the lifetime of # a PyTorchTrialContext. self.models = [] # type: List[nn.Module] self.optimizers = [] # type: List[torch.optim.Optimizer] self.profiler = None # type: Any self.lr_schedulers = [] # type: List[pytorch.LRScheduler] self._epoch_len = None # type: Optional[int] # Keep a map of wrapped models to their original input forms, which is needed # by torch DDP and apex to initialize in the correct order self._wrapped_models = {} # type: Dict[nn.Module, nn.Module] # Use a main model to contain all of the models because when using horovod # to broadcast the states of models we want to avoid name conflicts for these # states so we set all the models to be sub-module of the main model with # different names using __setattr__ and use the state_dict of the main model # for broadcasting. Note that broadcast_parameters only accepts state_dict() # although its doc says it also accepts named_parameters() self._main_model = nn.Module() self._scaler = None self._use_apex = False self._loss_ids = {} # type: Dict[torch.Tensor, int] self._last_backward_batch_idx = None # type: Optional[int] self._current_batch_idx = None # type: Optional[int] self.experimental = pytorch.PyTorchExperimentalContext(self) self._reducers = pytorch._PyTorchReducerContext() self._determined_profiler = None # type: Optional[profiler.ProfilerAgent] optimizations_config = self.env.experiment_config.get_optimizations_config( ) self._aggregation_frequency = cast( int, optimizations_config.get("aggregation_frequency")) self._fp16_compression = cast( bool, optimizations_config.get("gradient_compression")) self._average_aggregated_gradients = cast( bool, optimizations_config.get("average_aggregated_gradients")) self._average_training_metrics = cast( bool, optimizations_config.get("average_training_metrics"))
def __init__(self, *arg: Any, **kwarg: Any): super().__init__(*arg, **kwarg) self.dataset_initialized = False self._per_slot_batch_size, self._global_batch_size = util.calculate_batch_sizes( self.get_hparams(), self.env.experiment_config.slots_per_trial(), "TFKerasTrial", ) self.experimental = TFKerasExperimentalContext( self.env, self.distributed, self._per_slot_batch_size, ) # The following three attributes are initialized during the lifetime of a # TFKerasTrialContext instance by the user calling compile() and # fit_generator() / fit(), respectively. self.model = None # type: Optional[tf.keras.Model] self.compile_args = None # type: Optional[inspect.BoundArguments] self.train_config = None # type: Optional[TFKerasTrainConfig] optimizations_config = self.env.experiment_config.get_optimizations_config( ) self._aggregation_frequency = cast( int, optimizations_config.get("aggregation_frequency")) self._average_aggregated_gradients = cast( bool, optimizations_config.get("average_aggregated_gradients")) self._optimizers = [] # type: List[tf.keras.optimizers.Optimizer] self._wrapped_optimizers = [ ] # type: List[tf.keras.optimizers.Optimizer] self._compiled_optimizer = None # type: Optional[tf.keras.optimizers.Optimizer] # The following attributes may be configured via configure_fit(). Defaults match the # normal keras.fit() defaults. self._fit_verbose = True self._fit_class_weight = None self._fit_workers = 1 self._fit_use_multiprocessing = False self._fit_max_queue_size = 10 self._fit_shuffle = True self._fit_validation_steps = None