def __init__( self, scheduler: torch.optim.lr_scheduler._LRScheduler, step_mode: StepMode, ): """LRScheduler constructor Args: scheduler (:py:class:`torch.optim.lr_scheduler._LRScheduler`): Learning rate scheduler to be used by Determined. step_mode (:py:class:`det.pytorch.LRSchedulerStepMode`): The strategy Determined will use to call (or not call) scheduler.step(). 1. ``STEP_EVERY_EPOCH``: Determined will call scheduler.step() after every training epoch. No arguments will be passed to step(). 2. ``STEP_EVERY_BATCH``: Determined will call scheduler.step() after every training batch. No arguments will be passed to step(). 3. ``MANUAL_STEP``: Determined will not call scheduler.step() at all. It is up to the user to decide when to call scheduler.step(), and whether to pass any arguments. """ check.check_not_none(scheduler) check.check_isinstance(step_mode, LRScheduler.StepMode) self._scheduler = scheduler self._step_mode = step_mode
def __init__(self, scheduler: torch.optim.lr_scheduler._LRScheduler, step_mode: StepMode): """Wrapper for a PyTorch LRScheduler. Usage of this wrapper is required to properly schedule the optimizer's learning rate. This wrapper fulfills two main functions: 1. Save and restore the learning rate when a trial is paused, preempted, etc. 2. Step the learning rate scheduler at the configured frequency (e.g., every batch or every epoch). Args: scheduler (:py:class:`torch.optim.lr_scheduler._LRScheduler`): Learning rate scheduler to be used by Determined. step_mode (:py:class:`det.pytorch.LRSchedulerStepMode`): The strategy Determined will use to call (or not call) scheduler.step(). 1. ``STEP_EVERY_EPOCH``: Determined will call scheduler.step() after every training epoch. No arguments will be passed to step(). 2. ``STEP_EVERY_BATCH``: Determined will call scheduler.step() after every training batch. No arguments will be passed to step(). 3. ``MANUAL_STEP``: Determined will not call scheduler.step() at all. It is up to the user to decide when to call scheduler.step(), and whether to pass any arguments. """ check.check_not_none(scheduler) check.check_isinstance(step_mode, LRScheduler.StepMode) self.scheduler = scheduler self.step_mode = step_mode
def get_optimizer(self) -> torch.optim.Optimizer: # type: ignore """ Get the optimizer associated with the trial. This function should not be called from: * ``__init__`` * ``build_model()`` * ``optimizer()`` """ check.check_not_none(self.optimizer) return self.optimizer
def get_model(self) -> torch.nn.Module: """ Get the model associated with the trial. This function should not be called from: * ``__init__`` * ``build_model()`` """ check.check_not_none(self.model) return cast(torch.nn.Module, self.model)
def yield_checkpoint_model( self, wkld: workload.Workload, respond: workload.ResponseFunc) -> workload.Stream: start_time = _current_timestamp() # Only the chief container should checkpoint. if self.rendezvous_info.get_rank() != 0: respond(workload.Skipped()) return # Save the workload completed message for after checkpoint upload completes. message = None # type: Optional[workload.Response] def _respond(checkpoint_info: workload.Response) -> None: checkpoint_info = cast(Dict[str, Any], checkpoint_info) metadata = storage.StorageMetadata( storage_id, storage.StorageManager._list_directory(path), checkpoint_info.get("framework", ""), checkpoint_info.get("format", ""), ) logging.info("Saved trial to checkpoint {}".format( metadata.storage_id)) self.tensorboard_mgr.sync() nonlocal message message = { "type": "WORKLOAD_COMPLETED", "workload": wkld, "start_time": start_time, "end_time": _current_timestamp(), "metrics": metadata, } with self.storage_mgr.store_path() as (storage_id, path): yield wkld, [pathlib.Path(path)], _respond # Because the messaging is synchronous, the layer below us must have called _respond. check_not_none(message, "response function did not get called") message = cast(workload.Response, message) respond(message)
def main(args: List[str] = sys.argv[1:]) -> None: # TODO(#1690): Refactor admin command(s) to a separate CLI tool. if "DET_ADMIN" in os.environ: experiment_args_description.subs.append( Cmd( "delete", experiment.delete_experiment, "delete experiment", [ Arg("experiment_id", help="delete experiment"), Arg( "--yes", action="store_true", default=False, help="automatically answer yes to prompts", ), ], )) try: parser = make_parser() argcomplete.autocomplete(parser) parsed_args = parser.parse_args(args) def die(message: str, always_print_traceback: bool = False) -> None: if always_print_traceback or os.getenv( "DET_DEBUG", "").lower() in ("true", "1", "yes"): import traceback traceback.print_exc() parser.exit(1, colored(message + "\n", "red")) v = vars(parsed_args) if not v.get("func"): parser.print_usage() parser.exit(2, "{}: no subcommand specified\n".format(parser.prog)) cert_fn = str(auth.get_config_path().joinpath("master.crt")) if os.path.exists(cert_fn): os.environ["REQUESTS_CA_BUNDLE"] = cert_fn try: try: check_version(parsed_args) except requests.exceptions.SSLError: # An SSLError usually means that we queried a master over HTTPS and got an untrusted # cert, so allow the user to store and trust the current cert. (It could also mean # that we tried to talk HTTPS on the HTTP port, but distinguishing that based on the # exception is annoying, and we'll figure that out in the next step anyway.) addr = api.parse_master_address(parsed_args.master) check_not_none(addr.hostname) check_not_none(addr.port) try: cert_pem_data = ssl.get_server_certificate( (cast(str, addr.hostname), cast(int, addr.port))) except ssl.SSLError: die("Tried to connect over HTTPS but couldn't get a certificate from the " "master; consider using HTTP") cert_hash = hashlib.sha256( ssl.PEM_cert_to_DER_cert(cert_pem_data)).hexdigest() cert_fingerprint = ":".join(chunks(cert_hash, 2)) if not render.yes_or_no( "The master sent an untrusted certificate with this SHA256 fingerprint:\n" "{}\nDo you want to trust this certificate from now on?" .format(cert_fingerprint)): die("Unable to verify master certificate") with open(cert_fn, "w") as out: out.write(cert_pem_data) os.environ["REQUESTS_CA_BUNDLE"] = cert_fn check_version(parsed_args) parsed_args.func(parsed_args) except KeyboardInterrupt as e: raise e except (api.errors.BadRequestException, api.errors.BadResponseException) as e: die("Failed to {}: {}".format(parsed_args.func.__name__, e)) except api.errors.CorruptTokenCacheException: die("Failed to login: Attempted to read a corrupted token cache. " "The store has been deleted; please try again.") except Exception: die("Failed to {}".format(parsed_args.func.__name__), always_print_traceback=True) except KeyboardInterrupt: parser.exit(3, colored("Interrupting...\n", "red"))
def from_json(record: Dict[str, Any]) -> "StorageMetadata": check_not_none(record["uuid"], "Storage ID is undefined") check_not_none(record["resources"], "Resources are undefined") return StorageMetadata(record["uuid"], record["resources"], record.get("labels"))