def reindex(repo, finalize_only):
"""
Process runs left in 'in progress' state.
"""
repo_path = clean_repo_path(repo) or Repo.default_repo_path()
repo_status = Repo.check_repo_status(repo_path)
if repo_status != RepoStatus.UPDATED:
click.echo(f'\'{repo_path}\' is not updated. Cannot run indexing.')
repo_inst = Repo.from_path(repo_path)
runs_in_progress_dir = os.path.join(repo_inst.path, 'meta', 'progress')
runs_in_progress = set(os.listdir(runs_in_progress_dir))
if finalize_only:
if not runs_in_progress:
click.echo('Index is up to date.')
return
confirmed = click.confirm(f'This command will try to finalize all pending runs in aim repo located at '
f'\'{repo_path}\'. Do you want to proceed?')
if not confirmed:
return
else:
confirmed = click.confirm(f'This command will try to reindex all runs in aim repo located at '
f'\'{repo_path}\'. This process might take a while. Do you want to proceed?')
if not confirmed:
return
finalize_stalled_runs(repo_inst, runs=runs_in_progress)
if not finalize_only:
run_flushes_and_compactions(repo_inst, runs_to_skip=runs_in_progress)
def select_metrics(search_statement: str, repo_path: Optional[str] = None):
if repo_path is not None:
repo = Repo.from_path(repo_path)
else:
repo = Repo.default_repo()
if not repo:
return None
return repo.query_metrics(search_statement)
def select_runs(expression: Optional[str] = None, repo_path: Optional[str] = None):
if repo_path is not None:
repo = Repo.from_path(repo_path)
else:
repo = Repo.default_repo()
if not repo:
return None
return repo.query_runs(expression)
def __init__(self,
hashname: Optional[str] = None,
*,
repo: Optional[Union[str, 'Repo']] = None,
read_only: bool = False,
experiment: Optional[str] = None,
system_tracking_interval: int = DEFAULT_SYSTEM_TRACKING_INT):
hashname = hashname or generate_run_hash()
self._instance_creation_time = time()
if repo is None:
from aim.sdk.repo import Repo
repo = Repo.default_repo()
elif isinstance(repo, str):
from aim.sdk.repo import Repo
repo = Repo.from_path(repo)
self.repo = repo
self.read_only = read_only
if not read_only:
logger.debug(f'Opening Run {hashname} in write mode')
self.hashname = hashname
self._hash = None
self._props = None
self.contexts: Dict[Context, int] = dict()
if experiment:
self.props.experiment = experiment
self.meta_tree: TreeView = self.repo.request(
'meta', hashname, read_only=read_only,
from_union=True).tree().view('meta')
self.meta_run_tree: TreeView = self.meta_tree.view('chunks').view(
hashname)
self.meta_attrs_tree: TreeView = self.meta_tree.view('attrs')
self.meta_run_attrs_tree: TreeView = self.meta_run_tree.view('attrs')
self.series_run_tree: TreeView = self.repo.request(
'trcs', hashname, read_only=read_only).tree().view('trcs').view(
'chunks').view(hashname)
if not read_only:
# TODO: [AT] check this once Container db open locking is added
self.series_run_tree.preload()
self.series_counters: Dict[Tuple[Context, str], int] = Counter()
self._creation_time = None
self._system_resource_tracker: ResourceTracker = None
if not read_only:
self.props
self.creation_time
self._prepare_resource_tracker(system_tracking_interval)
def check_repo_integrity(repo: Repo, legacy_run_map: dict):
try:
ok = True
with click.progressbar(repo.iter_runs(),
item_show_func=lambda r:
(f'Checking run {r.hash}'
if r else '')) as runs:
for run in runs:
legacy_hash = run.get(['v2_params', 'run_hash'])
run_metrics = legacy_run_map.pop(legacy_hash)
for metric_name, ctx, _ in run.iter_metrics_info():
idx = run_metrics[metric_name].index(ctx.to_dict())
run_metrics[metric_name].pop(idx)
if not run_metrics[metric_name]:
del run_metrics[metric_name]
if run_metrics:
click.echo(
f'Run \'{run.hash}\' [\'{legacy_hash}\'] missing metrics \'{run_metrics.keys()}\'.'
)
ok = False
if legacy_run_map:
click.echo(f'Repo missing runs \'{legacy_run_map.keys()}\'.')
ok = False
except (KeyError, ValueError):
ok = False
if not ok:
raise RepoIntegrityError
def _init_test_repo():
repo = Repo.default_repo(init=True)
# some unittests check sequence tracking in a separate thread
# need to make sure task_queue is there
os.environ[AIM_ENABLE_TRACKING_THREAD] = 'ON'
Repo.tracking_queue = _get_tracking_queue()
del os.environ[AIM_ENABLE_TRACKING_THREAD]
def server(host, port, workers, repo, ssl_keyfile, ssl_certfile):
# TODO [MV, AT] remove code duplication with aim up cmd implementation
repo_path = clean_repo_path(repo) or Repo.default_repo_path()
repo_status = Repo.check_repo_status(repo_path)
if repo_status == RepoStatus.MISSING:
init_repo = click.confirm(
f'\'{repo_path}\' is not a valid Aim repository. Do you want to initialize it?'
)
if not init_repo:
click.echo('To initialize repo please run the following command:')
click.secho('aim init', fg='yellow')
return
repo_inst = Repo.from_path(repo_path, init=True)
elif repo_status == RepoStatus.UPDATE_REQUIRED:
upgrade_repo = click.confirm(
f'\'{repo_path}\' requires upgrade. Do you want to run upgrade automatically?'
)
if upgrade_repo:
from aim.cli.upgrade.utils import convert_2to3
repo_inst = convert_2to3(repo_path,
drop_existing=False,
skip_failed_runs=False,
skip_checks=False)
else:
click.echo('To upgrade repo please run the following command:')
click.secho(f'aim upgrade --repo {repo_path} 2to3', fg='yellow')
return
else:
repo_inst = Repo.from_path(repo_path)
if repo_status == RepoStatus.PATCH_REQUIRED:
repo_inst.structured_db.run_upgrades()
os.environ[AIM_SERVER_MOUNTED_REPO_PATH] = repo_inst.path
click.echo(
click.style('Running Aim Server on repo `{}`'.format(repo_inst),
fg='yellow'))
click.echo('Server is mounted on {}:{}'.format(host, port), err=True)
click.echo('Press Ctrl+C to exit')
try:
run_server(host, port, workers, ssl_keyfile, ssl_certfile)
except Exception:
click.echo('Failed to run Aim Tracking Server. '
'Please see the logs for details.')
return
def __init__(self):
root_path = get_root_path()
repo_path = f'{root_path}/{get_aim_repo_name()}'
self.name = 'My awesome project'
self.path = root_path
self.repo_path = repo_path
self.description = ''
self.repo = Repo.from_path(self.repo_path)
def _set_repo(self, repo):
if repo is None:
from aim.sdk.repo import Repo
repo = Repo.default_repo_path()
if isinstance(repo, str):
from aim.sdk.repo import Repo, RepoStatus
repo_status = Repo.check_repo_status(repo)
if repo_status == RepoStatus.UPDATE_REQUIRED:
logger.error(
f'Trying to start Run on repository {repo}, which is out of date. '
f'Please upgrade repository with the following command: '
f'`aim upgrade --repo {repo} 2to3`.')
raise RuntimeError()
elif repo_status == RepoStatus.MISSING:
repo = Repo.from_path(repo, init=True)
else:
repo = Repo.from_path(repo)
self.repo = repo
def __init__(self):
root_path = get_root_path()
repo_path = Project._default_repo_path or '{}/.aim'.format(root_path)
self.name = 'My awesome project'
self.path = root_path
self.repo_path = repo_path
self.description = ''
self.repo = Repo.from_path(self.repo_path)
def __init__(self, repo: Optional[str] = None,
experiment: Optional[str] = None,
flush_frequency: int = 0, # unused
block_termination: bool = True, # unused
run: Optional[str] = None,
system_tracking_interval: Optional[int] = DEFAULT_SYSTEM_TRACKING_INT):
self._repo = Repo.from_path(repo) if repo else Repo.default_repo()
self._repo_path = self._repo.path
self._run = Run(run, repo=self._repo, experiment=experiment,
system_tracking_interval=system_tracking_interval)
self._run_hash = self._run.hashname
self.active = True
Session.sessions.setdefault(self._repo_path, [])
Session.sessions[self._repo_path].append(self)
# Bind signal listeners
self._set_exit_handlers()
def list_runs(ctx):
repo_path = ctx.obj['repo']
if not Repo.exists(repo_path):
click.echo(f'\'{repo_path}\' is not a valid aim repo.')
exit(1)
chunks_dir = os.path.join(repo_path, '.aim', 'meta', 'chunks')
run_hashes = os.listdir(chunks_dir)
click.echo('\t'.join(run_hashes))
click.echo(f'Total {len(run_hashes)} runs.')
def remove_test_data():
repo = Repo.default_repo()
repo.container_pool.clear()
repo.container_view_pool.clear()
repo.persistent_pool.clear()
truncate_structured_db(repo.structured_db)
repo_path_base = repo.path
shutil.rmtree(os.path.join(repo_path_base, 'meta'), ignore_errors=True)
shutil.rmtree(os.path.join(repo_path_base, 'seqs'), ignore_errors=True)
shutil.rmtree(os.path.join(repo_path_base, 'locks'), ignore_errors=True)
shutil.rmtree(os.path.join(repo_path_base, 'progress'), ignore_errors=True)
def convert_2to3(path: str, drop_existing: bool = False, skip_failed_runs: bool = False, skip_checks: bool = False):
lrepo_path, repo_path = setup_directories(path)
def _rollback():
shutil.rmtree(repo_path, ignore_errors=True)
shutil.move(lrepo_path, repo_path)
try:
click.echo('Preparing new repository...')
shutil.move(repo_path, lrepo_path)
lrepo = LegacyRepo(mode=LegacyRepo.READING_MODE, repo_full_path=lrepo_path)
repo = Repo.from_path(repo_path, init=True)
repo.structured_db.run_upgrades()
click.echo('Analyzing legacy repository...')
lruns = collect_runs(lrepo)
click.echo(f'Collected {len(lruns)} runs.')
legacy_run_map = {}
with repo.structured_db:
with click.progressbar(lruns, show_pos=True,
item_show_func=lambda r: (f'Converting run {r.run_hash}' if r else '')) as legacy_runs:
for lrun in legacy_runs:
convert_run(lrun, repo, legacy_run_map, skip_failed=skip_failed_runs)
if not skip_checks:
click.echo('Checking repository integrity...')
check_repo_integrity(repo, legacy_run_map)
click.echo('Repository integrity check passed!')
else:
click.echo('WARN Skipping repository integrity checks.')
except KeyboardInterrupt:
click.echo('Convert interrupted by client. Rolling back...')
_rollback()
return
except RepoIntegrityError:
click.echo('Repository integrity check failed. Rolling back...', err=True)
click.echo('If you want to convert repository anyway, please rerun command with \'--skip-checks\' flag.')
_rollback()
return
except Exception as e:
click.echo(f'Failed to convert repository. '
f'Reason: {str(e)}')
_rollback()
return
if drop_existing:
click.echo('Removing legacy repository...')
shutil.rmtree(lrepo_path)
else:
click.echo(f'Legacy repository can be found at \'{lrepo_path}\'.')
click.echo('DONE')
def init():
"""
Initializes new repository in the current working directory:
- Creates .aim directory & runs upgrades for structured DB
"""
repo_path = os.getcwd()
re_init = False
if Repo.exists(repo_path):
re_init = click.confirm('Aim repository is already initialized. ' +
'Do you want to re-initialize it?')
if not re_init:
return
# Clear old repo
Repo.rm(repo_path)
repo = Repo.from_path(repo_path, init=True)
repo.structured_db.run_upgrades()
if re_init:
click.echo('Re-initialized empty Aim repository at {}'.format(
repo.root_path))
else:
click.echo(('Initialized a new ' + 'Aim repository at {}').format(
repo.root_path))
def init(repo):
"""
Initializes new repository in the --repo directory.
Initializes new repository in the current working directory if --repo argument is not provided:
- Creates .aim directory & runs upgrades for structured DB
"""
repo_path = clean_repo_path(repo) or os.getcwd()
re_init = False
if Repo.exists(repo_path):
re_init = click.confirm(
'Aim repository is already initialized. '
'Do you want to re-initialize to empty Aim repository?')
if not re_init:
return
# Clear old repo
Repo.rm(repo_path)
repo = Repo.from_path(repo_path, init=True)
if re_init:
click.echo('Re-initialized empty Aim repository at {}'.format(
repo.root_path))
else:
click.echo('Initialized a new Aim repository at {}'.format(
repo.root_path))
def remove_runs(ctx, hashes):
if len(hashes) == 0:
click.echo('Please specify at lest one Run to delete.')
exit(1)
repo_path = ctx.obj['repo']
confirmed = click.confirm(f'This command will permanently delete {len(hashes)} runs from aim repo located at '
f'\'{repo_path}\'. Do you want to proceed?')
if not confirmed:
return
repo = Repo.from_path(repo_path)
success, remaining_runs = repo.delete_runs(hashes)
if success:
click.echo(f'Successfully deleted {len(hashes)} runs.')
else:
click.echo('Something went wrong while deleting runs. Remaining runs are:', err=True)
click.secho('\t'.join(remaining_runs), fg='yellow')
def fill_up_test_data():
remove_test_data()
# put dummy data into test repo with 10 runs, tracking 2 metrics over 3 contexts
repo = Repo.default_repo()
run_hashes = [hex(random.getrandbits(64))[-7:] for _ in range(10)]
contexts = [{
'is_training': True,
'subset': 'train'
}, {
'is_training': True,
'subset': 'val'
}, {
'is_training': False
}]
metrics = ['loss', 'accuracy']
with repo.structured_db:
try:
for idx, hash_name in enumerate(run_hashes):
run = Run(hashname=hash_name,
repo=repo,
system_tracking_interval=None)
run['hparams'] = create_run_params()
run['run_index'] = idx
run['start_time'] = datetime.datetime.utcnow().isoformat()
run['name'] = f'Run # {idx}'
run.props.name = run['name']
metric_contexts = itertools.product(metrics, contexts)
for metric_context in metric_contexts:
metric = metric_context[0]
context = metric_context[1]
if metric == 'accuracy' and 'subset' in context:
continue
else:
# track 100 values per run
for step in range(100):
val = 1.0 - 1.0 / (step + 1)
run.track(val,
name=metric,
step=step,
epoch=1,
context=context)
finally:
del run
def setUpClass(cls) -> None:
super().setUpClass()
cls.repo = Repo.default_repo()
def setUpClass(cls) -> None:
fill_up_test_data()
cls.repo = Repo.default_repo()
def up(dev, host, port, workers, repo, tf_logs, ssl_keyfile, ssl_certfile,
base_path, force_init):
if dev:
os.environ[AIM_ENV_MODE_KEY] = 'dev'
else:
os.environ[AIM_ENV_MODE_KEY] = 'prod'
if base_path:
# process `base_path` as ui requires leading slash
if base_path.endswith('/'):
base_path = base_path[:-1]
if base_path and not base_path.startswith('/'):
base_path = f'/{base_path}'
os.environ[AIM_UI_BASE_PATH] = base_path
repo_path = clean_repo_path(repo) or Repo.default_repo_path()
repo_status = Repo.check_repo_status(repo_path)
if repo_status == RepoStatus.MISSING:
init_repo = None
if not force_init:
init_repo = click.confirm(
f'\'{repo_path}\' is not a valid Aim repository. Do you want to initialize it?'
)
else:
init_repo = True
if not init_repo:
click.echo('To initialize repo please run the following command:')
click.secho('aim init', fg='yellow')
return
repo_inst = Repo.from_path(repo_path, init=True)
elif repo_status == RepoStatus.UPDATE_REQUIRED:
upgrade_repo = click.confirm(
f'\'{repo_path}\' requires upgrade. Do you want to run upgrade automatically?'
)
if upgrade_repo:
from aim.cli.upgrade.utils import convert_2to3
repo_inst = convert_2to3(repo_path,
drop_existing=False,
skip_failed_runs=False,
skip_checks=False)
else:
click.echo('To upgrade repo please run the following command:')
click.secho(f'aim upgrade --repo {repo_path} 2to3', fg='yellow')
return
else:
repo_inst = Repo.from_path(repo_path)
if repo_status == RepoStatus.PATCH_REQUIRED:
repo_inst.structured_db.run_upgrades()
os.environ[AIM_UI_MOUNTED_REPO_PATH] = repo_inst.path
if tf_logs:
os.environ[AIM_TF_LOGS_PATH_KEY] = tf_logs
try:
db_cmd = build_db_upgrade_command()
exec_cmd(db_cmd, stream_output=True)
except ShellCommandException:
click.echo('Failed to initialize Aim DB. '
'Please see the logs above for details.')
return
if dev or (os.getenv(AIM_UI_TELEMETRY_KEY) is not None
and os.getenv(AIM_UI_TELEMETRY_KEY) == '0'):
os.environ[AIM_UI_TELEMETRY_KEY] = '0'
else:
os.environ[AIM_UI_TELEMETRY_KEY] = '1'
alert_msg = 'Aim UI collects anonymous usage analytics.'
opt_out_msg = 'Read how to opt-out here: '
opt_out_url = 'https://aimstack.readthedocs.io/en/latest/community/telemetry.html'
line_width = max(len(opt_out_msg), len(alert_msg),
len(opt_out_url)) + 8
click.echo('┌{}┐'.format('-' * (line_width - 2)))
click.echo('{}{}{}'.format(' ' * ((line_width - len(alert_msg)) // 2),
alert_msg,
' ' * ((line_width - len(alert_msg)) // 2)))
click.echo('{}{}{}'.format(
' ' * ((line_width - len(opt_out_msg)) // 2), opt_out_msg,
' ' * ((line_width - len(opt_out_msg)) // 2)))
click.echo('{}{}{}'.format(
' ' * ((line_width - len(opt_out_url)) // 2), opt_out_url,
' ' * ((line_width - len(opt_out_url)) // 2)))
click.echo('└{}┘'.format('-' * (line_width - 2)))
click.echo(
click.style('Running Aim UI on repo `{}`'.format(repo_inst),
fg='yellow'))
scheme = 'https' if ssl_keyfile or ssl_certfile else 'http'
click.echo('Open {}://{}:{}{}'.format(scheme, host, port, base_path),
err=True)
click.echo('Press Ctrl+C to exit')
try:
server_cmd = build_uvicorn_command(host, port, workers, ssl_keyfile,
ssl_certfile)
exec_cmd(server_cmd, stream_output=True)
except ShellCommandException:
click.echo('Failed to run Aim UI. '
'Please see the logs above for details.')
return
def up(dev, host, port, repo, tf_logs):
repo_path = clean_repo_path(repo)
if repo_path:
repo_inst = Repo.from_path(repo_path)
else:
repo_inst = Repo.default_repo()
repo_inst.structured_db.run_upgrades()
os.environ[AIM_UI_MOUNTED_REPO_PATH] = repo_inst.path
if dev:
os.environ[AIM_WEB_ENV_KEY] = 'dev'
else:
os.environ[AIM_WEB_ENV_KEY] = 'prod'
if tf_logs:
os.environ[AIM_TF_LOGS_PATH_KEY] = tf_logs
try:
db_cmd = build_db_upgrade_command()
exec_cmd(db_cmd, stream_output=True)
except ShellCommandException:
click.echo('Failed to initialize Aim DB. ' +
'Please see the logs above for details.')
return
if dev or (os.getenv(AIM_UI_TELEMETRY_KEY) is not None
and os.getenv(AIM_UI_TELEMETRY_KEY) == '0'):
os.environ[AIM_UI_TELEMETRY_KEY] = '0'
else:
os.environ[AIM_UI_TELEMETRY_KEY] = '1'
alert_msg = 'Aim UI collects anonymous usage analytics.'
opt_out_msg = 'Read how to opt-out here: '
opt_out_url = 'https://github.com/aimhubio/aim#anonymized-telemetry'
line_width = max(len(opt_out_msg), len(alert_msg)) + 16
click.echo('┌{}┐'.format('-' * (line_width - 2)))
click.echo('{}{}{}'.format(' ' * ((line_width - len(alert_msg)) // 2),
alert_msg,
' ' * ((line_width - len(alert_msg)) // 2)))
click.echo('{}{}{}'.format(
' ' * ((line_width - len(opt_out_msg)) // 2), opt_out_msg,
' ' * ((line_width - len(opt_out_msg)) // 2)))
click.echo('{}{}{}'.format(
' ' * ((line_width - len(opt_out_url)) // 2), opt_out_url,
' ' * ((line_width - len(opt_out_url)) // 2)))
click.echo('└{}┘'.format('-' * (line_width - 2)))
click.echo(
click.style('Running Aim UI on repo `{}`'.format(repo_inst),
fg='yellow'))
click.echo('Open http://{}:{}'.format(host, port))
click.echo('Press Ctrl+C to exit')
try:
server_cmd = build_uvicorn_command(host, port, 1)
exec_cmd(server_cmd, stream_output=True)
except ShellCommandException:
click.echo('Failed to run Aim UI. ' +
'Please see the logs above for details.')
return
