def list_remote_datasets(all_teams: bool, team: Optional[str] = None):
"""Lists remote datasets with its annotation progress"""
# TODO: add listing open datasets
table = Table(["name", "images"], [Table.L, Table.R])
datasets = []
if all_teams:
for team in _config().get_all_teams():
client = _load_client(team["slug"])
datasets += client.list_remote_datasets()
else:
client = _load_client(team)
datasets = client.list_remote_datasets()
for dataset in datasets:
table.add_row({"name": f"{dataset.team}/{dataset.slug}", "images": dataset.image_count})
if len(table) == 0:
print("No dataset available.")
else:
print(table)
def local():
"""Lists synced datasets, stored in the specified path. """
table = Table(["name", "images", "sync_date", "size"],
[Table.L, Table.R, Table.R, Table.R])
client = _load_client(offline=True)
for dataset_path in client.list_local_datasets():
table.add_row({
"name":
dataset_path.name,
"images":
sum(1 for _ in find_files([dataset_path])),
"sync_date":
humanize.naturaldate(
datetime.datetime.fromtimestamp(dataset_path.stat().st_mtime)),
"size":
humanize.naturalsize(
sum(p.stat().st_size for p in find_files([dataset_path]))),
})
print(table)
def local(team: Optional[str] = None):
"""Lists synced datasets, stored in the specified path. """
table = Table(["name", "images", "sync_date", "size"], [Table.L, Table.R, Table.R, Table.R])
client = _load_client(offline=True)
for dataset_path in client.list_local_datasets(team=team):
table.add_row(
{
"name": f"{dataset_path.parent.name}/{dataset_path.name}",
"images": sum(1 for _ in find_files([dataset_path])),
"sync_date": humanize.naturaldate(datetime.datetime.fromtimestamp(dataset_path.stat().st_mtime)),
"size": humanize.naturalsize(sum(p.stat().st_size for p in find_files([dataset_path]))),
}
)
# List deprecated datasets
deprecated_local_datasets = client.list_deprecated_local_datasets()
if deprecated_local_datasets:
for dataset_path in client.list_deprecated_local_datasets():
table.add_row(
{
"name": dataset_path.name + " (deprecated format)",
"images": sum(1 for _ in find_files([dataset_path])),
"sync_date": humanize.naturaldate(datetime.datetime.fromtimestamp(dataset_path.stat().st_mtime)),
"size": humanize.naturalsize(sum(p.stat().st_size for p in find_files([dataset_path]))),
}
)
print(table)
if len(list(deprecated_local_datasets)):
print(
f"\nWARNING: found some local datasets that use a deprecated format "
f"not supported by the recent version of darwin-py. "
f"Run `darwin dataset migrate team_slug/dataset_slug` "
"if you want to be able to use them in darwin-py."
)
def dataset_list_releases(dataset_slug: str):
client = _load_client(offline=False)
try:
dataset = client.get_remote_dataset(dataset_identifier=dataset_slug)
releases = dataset.get_releases()
if len(releases) == 0:
print("No available releases, export one first.")
return
table = Table(["name", "images", "classes", "export_date"],
[Table.L, Table.R, Table.R, Table.R])
for release in releases:
if not release.available:
continue
table.add_row({
"name": release.identifier,
"images": release.image_count,
"classes": release.class_count,
"export_date": release.export_date,
})
print(table)
except NotFound:
_error(f"No dataset with name '{dataset_slug}'")