def _set_strategy(self) -> None: if isinstance(self._strategy_flag, str) and self._strategy_flag in StrategyRegistry: self._strategy = StrategyRegistry.get(self._strategy_flag) if isinstance(self._strategy_flag, str): self.set_distributed_mode(self._strategy_flag) elif isinstance(self._strategy_flag, Strategy): self._strategy = self._strategy_flag
def test_strategy_registry_with_new_strategy(): class TestStrategy: distributed_backend = "test_strategy" def __init__(self, param1, param2): self.param1 = param1 self.param2 = param2 strategy_name = "test_strategy" strategy_description = "Test Strategy" StrategyRegistry.register(strategy_name, TestStrategy, description=strategy_description, param1="abc", param2=123) assert strategy_name in StrategyRegistry assert StrategyRegistry[strategy_name][ "description"] == strategy_description assert StrategyRegistry[strategy_name]["init_params"] == { "param1": "abc", "param2": 123 } assert StrategyRegistry[strategy_name][ "distributed_backend"] == "test_strategy" assert isinstance(StrategyRegistry.get(strategy_name), TestStrategy) StrategyRegistry.remove(strategy_name) assert strategy_name not in StrategyRegistry
def _init_strategy(self) -> None: """Instantiate the Strategy given depending on the setting of ``_strategy_flag``.""" if isinstance(self._strategy_flag, HorovodStrategy) or self._strategy_flag == "horovod": # handle horovod has to happen before initialize strategy because HorovodStrategy needs hvd.init() first. # TODO lazy initialized and setup horovod strategy `global_rank` self._handle_horovod() if isinstance(self._strategy_flag, str): self.strategy = StrategyRegistry.get(self._strategy_flag) elif isinstance(self._strategy_flag, Strategy): self.strategy = self._strategy_flag else: raise RuntimeError(f"{self.strategy} is not valid type: {self.strategy}")
def _init_strategy(self) -> None: """Instantiate the Strategy given depending on the setting of ``_strategy_flag``.""" if isinstance(self._strategy_flag, HorovodStrategy) or self._strategy_flag == "horovod": # handle horovod has to happen before initialize strategy because HorovodStrategy needs hvd.init() first. # TODO lazy initialized and setup horovod strategy `global_rank` self._handle_horovod() if isinstance(self._strategy_flag, str): if self._strategy_flag == "ddp2": # TODO: remove this error in v1.8 raise ValueError( "The DDP2 strategy is no longer supported. For single-node use, we recommend `strategy='ddp'` or" " `strategy='dp'` as a replacement. If you need DDP2, you will need `torch < 1.9`," " `pytorch-lightning < 1.5`, and set it as `accelerator='ddp2'`." ) self.strategy = StrategyRegistry.get(self._strategy_flag) elif isinstance(self._strategy_flag, Strategy): self.strategy = self._strategy_flag else: raise RuntimeError( f"{self.strategy} is not valid type: {self.strategy}")
def handle_given_plugins(self) -> None: for plug in self.plugins: if self._strategy_flag is not None and self._is_plugin_training_type(plug): raise MisconfigurationException( f"You have passed `Trainer(strategy={self._strategy_flag!r})`" f" and you can only specify one training type plugin, but you have passed {plug} as a plugin." ) if self._is_plugin_training_type(plug): rank_zero_deprecation( f"Passing {plug} `strategy` to the `plugins` flag in Trainer has been deprecated" f" in v1.5 and will be removed in v1.7. Use `Trainer(strategy={plug})` instead." ) strategy = self._strategy or None checkpoint = None precision = None cluster_environment = None for plug in self.plugins: if isinstance(plug, str) and plug in StrategyRegistry: if strategy is None: strategy = StrategyRegistry.get(plug) else: raise MisconfigurationException( "You can only specify one precision and one training type plugin." " Found more than 1 training type plugin:" f' {StrategyRegistry[plug]["strategy"]} registered to {plug}' ) if isinstance(plug, str): # Reset the distributed type as the user has overridden training type # via the plugins argument self._strategy_type = None self.set_distributed_mode(plug) elif isinstance(plug, Strategy): if strategy is None: strategy = plug else: raise MisconfigurationException( "You can only specify one training type plugin." f" Available: {type(strategy).__name__}, given: {type(plug).__name__}" ) elif isinstance(plug, PrecisionPlugin): if precision is None: precision = plug else: raise MisconfigurationException( "You can only specify one precision plugin." f" Available: {type(precision).__name__}, given: {type(plug).__name__}" ) elif isinstance(plug, CheckpointIO): if checkpoint is None: checkpoint = plug else: raise MisconfigurationException( "You can only specify one checkpoint plugin." f" Available: {type(checkpoint).__name__}, given: {type(plug).__name__}" ) elif isinstance(plug, ClusterEnvironment): if cluster_environment is None: cluster_environment = plug else: raise MisconfigurationException( "You can only specify one cluster environment. Found more than 1 cluster environment plugin" ) else: raise MisconfigurationException( f"Found invalid type for plugin {plug}. Expected a precision or training type plugin." ) self._strategy = strategy self._precision_plugin = precision self._checkpoint_io = checkpoint self._cluster_environment = cluster_environment