def get_source_code_to_upload( upload_source_files: Optional[List[str]], ) -> Tuple[str, List[Tuple[str, str]]]: source_target_pairs = [] if is_ipython(): main_file = None entrypoint = None else: main_file = sys.argv[0] entrypoint = main_file or None if upload_source_files is None: if main_file is not None and os.path.isfile(main_file): entrypoint = normalize_file_name(os.path.basename(main_file)) source_target_pairs = [( os.path.abspath(main_file), normalize_file_name(os.path.basename(main_file)), )] else: expanded_source_files = set() for filepath in upload_source_files: expanded_source_files |= set(glob(filepath)) if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 5): for filepath in expanded_source_files: if filepath.startswith(".."): raise ValueError( "You need to have Python 3.5 or later to use paths outside current directory." ) source_target_pairs.append( (os.path.abspath(filepath), normalize_file_name(filepath))) else: absolute_paths = [] for filepath in expanded_source_files: absolute_paths.append(os.path.abspath(filepath)) try: common_source_root = os.path.commonpath(absolute_paths) except ValueError: for absolute_path in absolute_paths: source_target_pairs.append( (absolute_path, normalize_file_name(absolute_path))) else: if os.path.isfile(common_source_root): common_source_root = os.path.dirname(common_source_root) if common_source_root.startswith(os.getcwd() + os.sep): common_source_root = os.getcwd() for absolute_path in absolute_paths: source_target_pairs.append(( absolute_path, normalize_file_name( os.path.relpath(absolute_path, common_source_root)), )) return entrypoint, source_target_pairs
def upload_source_code(source_files: Optional[List[str]], run: Run) -> None: if not is_ipython() and os.path.isfile(sys.argv[0]): if source_files is None: entrypoint = os.path.basename(sys.argv[0]) source_files = sys.argv[0] elif not source_files: entrypoint = os.path.basename(sys.argv[0]) else: common_root = get_common_root(get_absolute_paths(source_files)) if common_root is not None: entrypoint = normalize_file_name( os.path.relpath(os.path.abspath(sys.argv[0]), common_root) ) else: entrypoint = normalize_file_name(os.path.abspath(sys.argv[0])) run[attr_consts.SOURCE_CODE_ENTRYPOINT_ATTRIBUTE_PATH] = entrypoint if source_files is not None: run[attr_consts.SOURCE_CODE_FILES_ATTRIBUTE_PATH].upload_files(source_files)
def create_checkpoint(backend, notebook_id, notebook_path): if is_ipython(): # pylint:disable=bad-option-value,import-outside-toplevel,import-error import IPython ipython = IPython.core.getipython.get_ipython() execution_count = -1 if ipython.kernel is not None: execution_count = ipython.kernel.execution_count with _checkpoints_lock: if execution_count in _checkpoints: return _checkpoints[execution_count] checkpoint = backend.create_checkpoint(notebook_id, notebook_path) if ipython is not None and ipython.kernel is not None: send_checkpoint_created(notebook_id=notebook_id, notebook_path=notebook_path, checkpoint_id=checkpoint.id) _checkpoints[execution_count] = checkpoint return checkpoint
def _run_aborting_thread(self, abort_callback): if abort_callback is not None: abort_impl = CustomAbortImpl(abort_callback) elif not is_ipython(): abort_impl = DefaultAbortImpl(pid=os.getpid()) else: return websocket_factory = self._backend.websockets_factory( # pylint: disable=protected-access project_id=self._experiment._project.internal_id, experiment_id=self._experiment.internal_id, ) if not websocket_factory: return self._aborting_thread = AbortingThread( websocket_factory=websocket_factory, abort_impl=abort_impl, experiment=self._experiment, ) self._aborting_thread.start()
def create_experiment(self, name=None, description=None, params=None, properties=None, tags=None, upload_source_files=None, abort_callback=None, logger=None, upload_stdout=True, upload_stderr=True, send_hardware_metrics=True, run_monitoring_thread=True, handle_uncaught_exceptions=True, git_info=None, hostname=None, notebook_id=None, notebook_path=None): """Create and start Neptune experiment. Create experiment, set its status to `running` and append it to the top of the experiments view. All parameters are optional, hence minimal invocation: ``neptune.create_experiment()``. Args: name (:obj:`str`, optional, default is ``'Untitled'``): Editable name of the experiment. Name is displayed in the experiment's `Details` (`Metadata` section) and in `experiments view` as a column. description (:obj:`str`, optional, default is ``''``): Editable description of the experiment. Description is displayed in the experiment's `Details` (`Metadata` section) and can be displayed in the `experiments view` as a column. params (:obj:`dict`, optional, default is ``{}``): Parameters of the experiment. After experiment creation ``params`` are read-only (see: :meth:`~neptune.experiments.Experiment.get_parameters`). Parameters are displayed in the experiment's `Details` (`Parameters` section) and each key-value pair can be viewed in `experiments view` as a column. properties (:obj:`dict`, optional, default is ``{}``): Properties of the experiment. They are editable after experiment is created. Properties are displayed in the experiment's `Details` (`Properties` section) and each key-value pair can be viewed in `experiments view` as a column. tags (:obj:`list`, optional, default is ``[]``): Must be list of :obj:`str`. Tags of the experiment. They are editable after experiment is created (see: :meth:`~neptune.experiments.Experiment.append_tag` and :meth:`~neptune.experiments.Experiment.remove_tag`). Tags are displayed in the experiment's `Details` (`Metadata` section) and can be viewed in `experiments view` as a column. upload_source_files (:obj:`list` or :obj:`str`, optional, default is ``None``): List of source files to be uploaded. Must be list of :obj:`str` or single :obj:`str`. Uploaded sources are displayed in the experiment's `Source code` tab. | If ``None`` is passed, Python file from which experiment was created will be uploaded. | Pass empty list (``[]``) to upload no files. | Unix style pathname pattern expansion is supported. For example, you can pass ``'*.py'`` to upload all python source files from the current directory. For Python 3.5 or later, paths of uploaded files on server are resolved as relative to the | calculated common root of all uploaded source files. For older Python versions, paths on server are | resolved always as relative to the current directory. For recursion lookup use ``'**/*.py'`` (for Python 3.5 and later). For more information see `glob library <https://docs.python.org/3/library/glob.html>`_. abort_callback (:obj:`callable`, optional, default is ``None``): Callback that defines how `abort experiment` action in the Web application should work. Actual behavior depends on your setup: * (default) If ``abort_callback=None`` and `psutil <https://psutil.readthedocs.io/en/latest/>`_ is installed, then current process and it's children are aborted by sending `SIGTERM`. If, after grace period, processes are not terminated, `SIGKILL` is sent. * If ``abort_callback=None`` and `psutil <https://psutil.readthedocs.io/en/latest/>`_ is **not** installed, then `abort experiment` action just marks experiment as *aborted* in the Web application. No action is performed on the current process. * If ``abort_callback=callable``, then ``callable`` is executed when `abort experiment` action in the Web application is triggered. logger (:obj:`logging.handlers` or `None`, optional, default is ``None``): If `handler <https://docs.python.org/3.6/library/logging.handlers.html>`_ to `Python logger` is passed, new experiment's `text log` (see: :meth:`~neptune.experiments.Experiment.log_text`) with name `"logger"` is created. Each time `Python logger` logs new data, it is automatically sent to the `"logger"` in experiment. As a results all data from `Python logger` are in the `Logs` tab in the experiment. upload_stdout (:obj:`Boolean`, optional, default is ``True``): Whether to send stdout to experiment's *Monitoring*. upload_stderr (:obj:`Boolean`, optional, default is ``True``): Whether to send stderr to experiment's *Monitoring*. send_hardware_metrics (:obj:`Boolean`, optional, default is ``True``): Whether to send hardware monitoring logs (CPU, GPU, Memory utilization) to experiment's *Monitoring*. run_monitoring_thread (:obj:`Boolean`, optional, default is ``True``): Whether to run thread that pings Neptune server in order to determine if experiment is responsive. handle_uncaught_exceptions (:obj:`Boolean`, optional, default is ``True``): Two options ``True`` and ``False`` are possible: * If set to ``True`` and uncaught exception occurs, then Neptune automatically place `Traceback` in the experiment's `Details` and change experiment status to `Failed`. * If set to ``False`` and uncaught exception occurs, then no action is performed in the Web application. As a consequence, experiment's status is `running` or `not responding`. git_info (:class:`~neptune.git_info.GitInfo`, optional, default is ``None``): | Instance of the class :class:`~neptune.git_info.GitInfo` that provides information about the git repository from which experiment was started. | If ``None`` is passed, system attempts to automatically extract information about git repository in the following way: * System looks for `.git` file in the current directory and, if not found, goes up recursively until `.git` file will be found (see: :meth:`~neptune.utils.get_git_info`). * If there is no git repository, then no information about git is displayed in experiment details in Neptune web application. hostname (:obj:`str`, optional, default is ``None``): If ``None``, neptune automatically get `hostname` information. User can also set `hostname` directly by passing :obj:`str`. Returns: :class:`~neptune.experiments.Experiment` object that is used to manage experiment and log data to it. Raises: `ExperimentValidationError`: When provided arguments are invalid. `ExperimentLimitReached`: When experiment limit in the project has been reached. Examples: .. code:: python3 # minimal invoke neptune.create_experiment() # explicitly return experiment object experiment = neptune.create_experiment() # create experiment with name and two parameters neptune.create_experiment(name='first-pytorch-ever', params={'lr': 0.0005, 'dropout': 0.2}) # create experiment with name and description, and no sources files uploaded neptune.create_experiment(name='neural-net-mnist', description='neural net trained on MNIST', upload_source_files=[]) # Send all py files in cwd (excluding hidden files with names beginning with a dot) neptune.create_experiment(upload_source_files='*.py') # Send all py files from all subdirectories (excluding hidden files with names beginning with a dot) # Supported on Python 3.5 and later. neptune.create_experiment(upload_source_files='**/*.py') # Send all files and directories in cwd (excluding hidden files with names beginning with a dot) neptune.create_experiment(upload_source_files='*') # Send all files and directories in cwd including hidden files neptune.create_experiment(upload_source_files=['*', '.*']) # Send files with names being a single character followed by '.py' extension. neptune.create_experiment(upload_source_files='?.py') # larger example neptune.create_experiment(name='first-pytorch-ever', params={'lr': 0.0005, 'dropout': 0.2}, properties={'key1': 'value1', 'key2': 17, 'key3': 'other-value'}, description='write longer description here', tags=['list-of', 'tags', 'goes-here', 'as-list-of-strings'], upload_source_files=['training_with_pytorch.py', 'net.py']) """ if name is None: name = "Untitled" if description is None: description = "" if params is None: params = {} if properties is None: properties = {} if tags is None: tags = [] if git_info is None: git_info = get_git_info(discover_git_repo_location()) if hostname is None: hostname = get_hostname() if notebook_id is None and os.getenv(NOTEBOOK_ID_ENV_NAME, None) is not None: notebook_id = os.environ[NOTEBOOK_ID_ENV_NAME] if isinstance(upload_source_files, six.string_types): upload_source_files = [upload_source_files] upload_source_entries = [] if is_ipython(): main_file = None entrypoint = None else: main_file = sys.argv[0] entrypoint = main_file or None if upload_source_files is None: if main_file is not None and os.path.isfile(main_file): entrypoint = normalize_file_name(os.path.basename(main_file)) upload_source_entries = [ UploadEntry(os.path.abspath(main_file), normalize_file_name(os.path.basename(main_file))) ] else: expanded_source_files = set() for filepath in upload_source_files: expanded_source_files |= set(glob(filepath)) if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 5): for filepath in expanded_source_files: if filepath.startswith('..'): raise ValueError('You need to have Python 3.5 or later to use paths outside current directory.') upload_source_entries.append(UploadEntry(os.path.abspath(filepath), normalize_file_name(filepath))) else: absolute_paths = [] for filepath in expanded_source_files: absolute_paths.append(os.path.abspath(filepath)) try: common_source_root = os.path.commonpath(absolute_paths) except ValueError: for absolute_path in absolute_paths: upload_source_entries.append(UploadEntry(absolute_path, normalize_file_name(absolute_path))) else: if os.path.isfile(common_source_root): common_source_root = os.path.dirname(common_source_root) if common_source_root.startswith(os.getcwd() + os.sep): common_source_root = os.getcwd() for absolute_path in absolute_paths: upload_source_entries.append(UploadEntry(absolute_path, normalize_file_name( os.path.relpath(absolute_path, common_source_root)))) if notebook_path is None and os.getenv(NOTEBOOK_PATH_ENV_NAME, None) is not None: notebook_path = os.environ[NOTEBOOK_PATH_ENV_NAME] abortable = abort_callback is not None or DefaultAbortImpl.requirements_installed() checkpoint_id = None if notebook_id is not None and notebook_path is not None: checkpoint = create_checkpoint(backend=self._backend, notebook_id=notebook_id, notebook_path=notebook_path) if checkpoint is not None: checkpoint_id = checkpoint.id experiment = self._backend.create_experiment( project=self, name=name, description=description, params=params, properties=properties, tags=tags, abortable=abortable, monitored=run_monitoring_thread, git_info=git_info, hostname=hostname, entrypoint=entrypoint, notebook_id=notebook_id, checkpoint_id=checkpoint_id ) # pylint: disable=protected-access experiment._start( upload_source_entries=upload_source_entries, abort_callback=abort_callback, logger=logger, upload_stdout=upload_stdout, upload_stderr=upload_stderr, send_hardware_metrics=send_hardware_metrics, run_monitoring_thread=run_monitoring_thread, handle_uncaught_exceptions=handle_uncaught_exceptions ) self._push_new_experiment(experiment) click.echo(self._get_experiment_link(experiment)) return experiment