def rm(cls, path: str): path = clean_repo_path(path) repo = cls._pool.get(path) if repo is not None: del cls._pool[path] aim_repo_path = os.path.join(path, AIM_REPO_NAME) shutil.rmtree(aim_repo_path)
def get_version(cls, path: str): path = clean_repo_path(path) version_file_path = os.path.join(path, get_aim_repo_name(), 'VERSION') if os.path.exists(version_file_path): with open(version_file_path, 'r') as version_fh: return version_fh.read() return '0.0' # old Aim repos
def from_path(cls, path: str, read_only: bool = None, init: bool = False): path = clean_repo_path(path) repo = cls._pool.get(path) if repo is None: repo = Repo(path, read_only=read_only, init=init) cls._pool[path] = repo return repo
def reindex(repo, finalize_only): """ Process runs left in 'in progress' state. """ repo_path = clean_repo_path(repo) or Repo.default_repo_path() repo_status = Repo.check_repo_status(repo_path) if repo_status != RepoStatus.UPDATED: click.echo(f'\'{repo_path}\' is not updated. Cannot run indexing.') repo_inst = Repo.from_path(repo_path) runs_in_progress_dir = os.path.join(repo_inst.path, 'meta', 'progress') runs_in_progress = set(os.listdir(runs_in_progress_dir)) if finalize_only: if not runs_in_progress: click.echo('Index is up to date.') return confirmed = click.confirm(f'This command will try to finalize all pending runs in aim repo located at ' f'\'{repo_path}\'. Do you want to proceed?') if not confirmed: return else: confirmed = click.confirm(f'This command will try to reindex all runs in aim repo located at ' f'\'{repo_path}\'. This process might take a while. Do you want to proceed?') if not confirmed: return finalize_stalled_runs(repo_inst, runs=runs_in_progress) if not finalize_only: run_flushes_and_compactions(repo_inst, runs_to_skip=runs_in_progress)
def exists(cls, path: str) -> bool: """Check Aim repository existence. Args: path (str): Path to Aim repository. Returns: True if repository exists, False otherwise. """ path = clean_repo_path(path) aim_repo_path = os.path.join(path, get_aim_repo_name()) return os.path.exists(aim_repo_path)
def rm(cls, path: str): """Remove Aim repository. Args: path (str): Path to Aim repository. """ path = clean_repo_path(path) repo = cls._pool.get(path) if repo is not None: del cls._pool[path] aim_repo_path = os.path.join(path, get_aim_repo_name()) shutil.rmtree(aim_repo_path)
def setup_directories(input_path: str): path = clean_repo_path(input_path) repo_path = path + '/.aim' new_repo_path = path + '/.aim_progress' l_repo_path = path + '/.aim_legacy' if not (os.path.exists(repo_path) and os.path.isdir(repo_path)): click.echo( f'\'{input_path}\' does not seem to be an Aim repository. Exiting.' ) exit(1) if os.path.exists(new_repo_path): shutil.rmtree(new_repo_path, ignore_errors=True) os.environ[AIM_REPO_NAME] = '.aim_progress' return repo_path, new_repo_path, l_repo_path
def __init__(self, path=None, repo_branch=None, repo_commit=None, repo_full_path=None, mode=WRITING_MODE): self._config = {} path = clean_repo_path(path) self.path = repo_full_path or os.path.join(path, AIM_REPO_NAME) self.config_path = os.path.join(self.path, AIM_CONFIG_FILE_NAME) self.hash = hashlib.md5(self.path.encode('utf-8')).hexdigest() self.active_commit = repo_commit or AIM_COMMIT_INDEX_DIR_NAME if re.match(r'^[A-Za-z0-9_\-]{2,}$', self.active_commit) is None: raise ValueError('run name must be at least 2 characters ' + 'and contain only latin letters, numbers, ' + 'dash and underscore') self.root_path = repo_full_path or path self.name = self.root_path.split(os.sep)[-1] self.branch_path = None self.index_path = None self.objects_dir_path = None self.media_dir_path = None self.records_storage = None self.mode = mode active_exp = self.config.get('active_branch') if repo_branch is not None: experiment = repo_branch elif active_exp is not None: experiment = active_exp else: experiment = None if experiment is not None: run_full_path = get_experiment_run_path(self.path, experiment, self.active_commit) else: run_full_path = None if self.active_commit != AIM_COMMIT_INDEX_DIR_NAME and run_full_path \ and os.path.exists(run_full_path): raise ValueError( ('run `{}` already exists' + '').format(self.active_commit))
def server(host, port, workers, repo, ssl_keyfile, ssl_certfile): # TODO [MV, AT] remove code duplication with aim up cmd implementation repo_path = clean_repo_path(repo) or Repo.default_repo_path() repo_status = Repo.check_repo_status(repo_path) if repo_status == RepoStatus.MISSING: init_repo = click.confirm( f'\'{repo_path}\' is not a valid Aim repository. Do you want to initialize it?' ) if not init_repo: click.echo('To initialize repo please run the following command:') click.secho('aim init', fg='yellow') return repo_inst = Repo.from_path(repo_path, init=True) elif repo_status == RepoStatus.UPDATE_REQUIRED: upgrade_repo = click.confirm( f'\'{repo_path}\' requires upgrade. Do you want to run upgrade automatically?' ) if upgrade_repo: from aim.cli.upgrade.utils import convert_2to3 repo_inst = convert_2to3(repo_path, drop_existing=False, skip_failed_runs=False, skip_checks=False) else: click.echo('To upgrade repo please run the following command:') click.secho(f'aim upgrade --repo {repo_path} 2to3', fg='yellow') return else: repo_inst = Repo.from_path(repo_path) if repo_status == RepoStatus.PATCH_REQUIRED: repo_inst.structured_db.run_upgrades() os.environ[AIM_SERVER_MOUNTED_REPO_PATH] = repo_inst.path click.echo( click.style('Running Aim Server on repo `{}`'.format(repo_inst), fg='yellow')) click.echo('Server is mounted on {}:{}'.format(host, port), err=True) click.echo('Press Ctrl+C to exit') try: run_server(host, port, workers, ssl_keyfile, ssl_certfile) except Exception: click.echo('Failed to run Aim Tracking Server. ' 'Please see the logs for details.') return
def from_path(cls, path: str, read_only: bool = None, init: bool = False): """Named constructor for Repo for given path. Arguments: path (str): Path to Aim repository. read_only (:obj:`bool`, optional): Flag for opening Repo in readonly mode. False by default. init (:obj:`bool`, optional): Flag used to initialize new Repo. False by default. Recommended to use ``aim init`` command instead. Returns: :obj:`Repo` object. """ if not path.startswith('ssh://') and not path.startswith('aim://'): path = clean_repo_path(path) repo = cls._pool.get(path) if repo is None: repo = Repo(path, read_only=read_only, init=init) cls._pool[path] = repo return repo
def init(repo): """ Initializes new repository in the --repo directory. Initializes new repository in the current working directory if --repo argument is not provided: - Creates .aim directory & runs upgrades for structured DB """ repo_path = clean_repo_path(repo) or os.getcwd() re_init = False if Repo.exists(repo_path): re_init = click.confirm( 'Aim repository is already initialized. ' 'Do you want to re-initialize to empty Aim repository?') if not re_init: return # Clear old repo Repo.rm(repo_path) repo = Repo.from_path(repo_path, init=True) if re_init: click.echo('Re-initialized empty Aim repository at {}'.format( repo.root_path)) else: click.echo('Initialized a new Aim repository at {}'.format( repo.root_path))
def get_root_path(): return clean_repo_path(os.getenv(AIM_UI_MOUNTED_REPO_PATH, os.getcwd()))
def setup_directories(path: str): path = clean_repo_path(path) repo_path = path + '/.aim' lrepo_path = path + '/.aim_legacy' return lrepo_path, repo_path
def up(dev, host, port, workers, repo, tf_logs, ssl_keyfile, ssl_certfile, base_path, force_init): if dev: os.environ[AIM_ENV_MODE_KEY] = 'dev' else: os.environ[AIM_ENV_MODE_KEY] = 'prod' if base_path: # process `base_path` as ui requires leading slash if base_path.endswith('/'): base_path = base_path[:-1] if base_path and not base_path.startswith('/'): base_path = f'/{base_path}' os.environ[AIM_UI_BASE_PATH] = base_path repo_path = clean_repo_path(repo) or Repo.default_repo_path() repo_status = Repo.check_repo_status(repo_path) if repo_status == RepoStatus.MISSING: init_repo = None if not force_init: init_repo = click.confirm( f'\'{repo_path}\' is not a valid Aim repository. Do you want to initialize it?' ) else: init_repo = True if not init_repo: click.echo('To initialize repo please run the following command:') click.secho('aim init', fg='yellow') return repo_inst = Repo.from_path(repo_path, init=True) elif repo_status == RepoStatus.UPDATE_REQUIRED: upgrade_repo = click.confirm( f'\'{repo_path}\' requires upgrade. Do you want to run upgrade automatically?' ) if upgrade_repo: from aim.cli.upgrade.utils import convert_2to3 repo_inst = convert_2to3(repo_path, drop_existing=False, skip_failed_runs=False, skip_checks=False) else: click.echo('To upgrade repo please run the following command:') click.secho(f'aim upgrade --repo {repo_path} 2to3', fg='yellow') return else: repo_inst = Repo.from_path(repo_path) if repo_status == RepoStatus.PATCH_REQUIRED: repo_inst.structured_db.run_upgrades() os.environ[AIM_UI_MOUNTED_REPO_PATH] = repo_inst.path if tf_logs: os.environ[AIM_TF_LOGS_PATH_KEY] = tf_logs try: db_cmd = build_db_upgrade_command() exec_cmd(db_cmd, stream_output=True) except ShellCommandException: click.echo('Failed to initialize Aim DB. ' 'Please see the logs above for details.') return if dev or (os.getenv(AIM_UI_TELEMETRY_KEY) is not None and os.getenv(AIM_UI_TELEMETRY_KEY) == '0'): os.environ[AIM_UI_TELEMETRY_KEY] = '0' else: os.environ[AIM_UI_TELEMETRY_KEY] = '1' alert_msg = 'Aim UI collects anonymous usage analytics.' opt_out_msg = 'Read how to opt-out here: ' opt_out_url = 'https://aimstack.readthedocs.io/en/latest/community/telemetry.html' line_width = max(len(opt_out_msg), len(alert_msg), len(opt_out_url)) + 8 click.echo('┌{}┐'.format('-' * (line_width - 2))) click.echo('{}{}{}'.format(' ' * ((line_width - len(alert_msg)) // 2), alert_msg, ' ' * ((line_width - len(alert_msg)) // 2))) click.echo('{}{}{}'.format( ' ' * ((line_width - len(opt_out_msg)) // 2), opt_out_msg, ' ' * ((line_width - len(opt_out_msg)) // 2))) click.echo('{}{}{}'.format( ' ' * ((line_width - len(opt_out_url)) // 2), opt_out_url, ' ' * ((line_width - len(opt_out_url)) // 2))) click.echo('└{}┘'.format('-' * (line_width - 2))) click.echo( click.style('Running Aim UI on repo `{}`'.format(repo_inst), fg='yellow')) scheme = 'https' if ssl_keyfile or ssl_certfile else 'http' click.echo('Open {}://{}:{}{}'.format(scheme, host, port, base_path), err=True) click.echo('Press Ctrl+C to exit') try: server_cmd = build_uvicorn_command(host, port, workers, ssl_keyfile, ssl_certfile) exec_cmd(server_cmd, stream_output=True) except ShellCommandException: click.echo('Failed to run Aim UI. ' 'Please see the logs above for details.') return
def up(dev, host, port, repo, tf_logs): repo_path = clean_repo_path(repo) if repo_path: repo_inst = Repo.from_path(repo_path) else: repo_inst = Repo.default_repo() repo_inst.structured_db.run_upgrades() os.environ[AIM_UI_MOUNTED_REPO_PATH] = repo_inst.path if dev: os.environ[AIM_WEB_ENV_KEY] = 'dev' else: os.environ[AIM_WEB_ENV_KEY] = 'prod' if tf_logs: os.environ[AIM_TF_LOGS_PATH_KEY] = tf_logs try: db_cmd = build_db_upgrade_command() exec_cmd(db_cmd, stream_output=True) except ShellCommandException: click.echo('Failed to initialize Aim DB. ' + 'Please see the logs above for details.') return if dev or (os.getenv(AIM_UI_TELEMETRY_KEY) is not None and os.getenv(AIM_UI_TELEMETRY_KEY) == '0'): os.environ[AIM_UI_TELEMETRY_KEY] = '0' else: os.environ[AIM_UI_TELEMETRY_KEY] = '1' alert_msg = 'Aim UI collects anonymous usage analytics.' opt_out_msg = 'Read how to opt-out here: ' opt_out_url = 'https://github.com/aimhubio/aim#anonymized-telemetry' line_width = max(len(opt_out_msg), len(alert_msg)) + 16 click.echo('┌{}┐'.format('-' * (line_width - 2))) click.echo('{}{}{}'.format(' ' * ((line_width - len(alert_msg)) // 2), alert_msg, ' ' * ((line_width - len(alert_msg)) // 2))) click.echo('{}{}{}'.format( ' ' * ((line_width - len(opt_out_msg)) // 2), opt_out_msg, ' ' * ((line_width - len(opt_out_msg)) // 2))) click.echo('{}{}{}'.format( ' ' * ((line_width - len(opt_out_url)) // 2), opt_out_url, ' ' * ((line_width - len(opt_out_url)) // 2))) click.echo('└{}┘'.format('-' * (line_width - 2))) click.echo( click.style('Running Aim UI on repo `{}`'.format(repo_inst), fg='yellow')) click.echo('Open http://{}:{}'.format(host, port)) click.echo('Press Ctrl+C to exit') try: server_cmd = build_uvicorn_command(host, port, 1) exec_cmd(server_cmd, stream_output=True) except ShellCommandException: click.echo('Failed to run Aim UI. ' + 'Please see the logs above for details.') return
def exists(cls, path: str) -> bool: path = clean_repo_path(path) aim_repo_path = os.path.join(path, AIM_REPO_NAME) return os.path.exists(aim_repo_path)