def fetch_data(repo: Repository, remote, startpoint, column, nbytes, all_): """Get data from REMOTE referenced by STARTPOINT (short-commit or branch). The default behavior is to only download a single commit's data or the HEAD commit of a branch. Please review optional arguments for other behaviors. """ from hangar.records.commiting import expand_short_commit_digest from hangar.records.heads import get_branch_head_commit from hangar.records.heads import get_staging_branch_head from hangar.utils import parse_bytes if startpoint is None: branch = get_staging_branch_head(repo._env.branchenv) commit = get_branch_head_commit(repo._env.branchenv, branch) elif startpoint in repo.list_branches(): commit = get_branch_head_commit(repo._env.branchenv, startpoint) else: commit = expand_short_commit_digest(repo._env.refenv, startpoint) click.echo(f'Fetching data for commit: {commit}') try: max_nbytes = parse_bytes(nbytes) except AttributeError: max_nbytes = None if len(column) == 0: column = None commits = repo.remote.fetch_data(remote=remote, commit=commit, column_names=column, max_num_bytes=max_nbytes, retrieve_all_history=all_) click.echo(f'completed data for commits: {commits}')
def test_check_repo_size(repo_20_filled_samples): from hangar.utils import parse_bytes, folder_size expected_nbytes = folder_size(repo_20_filled_samples._repo_path, recurse=True) nbytes = repo_20_filled_samples.size_nbytes assert expected_nbytes == nbytes format_nbytes = repo_20_filled_samples.size_human # account for rounding when converting int to str. assert nbytes * 0.95 <= parse_bytes(format_nbytes) <= nbytes * 1.05
def fetch_data(ctx, remote, startpoint, aset, nbytes, all_): """Get data from REMOTE referenced by STARTPOINT (short-commit or branch). The default behavior is to only download a single commit's data or the HEAD commit of a branch. Please review optional arguments for other behaviors """ from hangar.records.heads import get_branch_head_commit, get_staging_branch_head from hangar.utils import parse_bytes P = os.getcwd() repo = Repository(path=P) if startpoint is None: branch = get_staging_branch_head(repo._env.branchenv) commit = get_branch_head_commit(repo._env.branchenv, branch) click.echo( f'No startpoint supplied, fetching data of HEAD: {commit} for BRANCH: {branch}' ) elif startpoint in repo.list_branches(): commit = get_branch_head_commit(repo._env.branchenv, startpoint) click.echo( f'Fetching data for HEAD: {commit} of STARTPOINT BRANCH: {startpoint}' ) else: commit = expand_short_commit_digest(repo._env.refenv, startpoint) click.echo(f'Fetching data for STARTPOINT HEAD: {commit}') click.echo(f'aset argument: {aset}') try: max_nbytes = parse_bytes(nbytes) click.echo(f'nbytes argument: {max_nbytes}') except AttributeError: max_nbytes = None if len(aset) == 0: aset = None commits = repo.remote.fetch_data(remote=remote, commit=commit, arrayset_names=aset, max_num_bytes=max_nbytes, retrieve_all_history=all_) click.echo(f'completed data for commits: {commits}')
def test_parse_bytes(arg, expected): from hangar.utils import parse_bytes res = parse_bytes(arg) assert res == expected