def is_image_file(ctx, param, value):
for filename in value:
if not LocalDataset.get_image_info(filename):
raise click.BadParameter(
f"{filename} is not an image -- use 'cvc publish' to add associated files"
)
return value
def wrapper(*args, **kwargs):
ctx_obj = get_current_context().obj
path = ctx_obj["cvc_local_path"]
conservator = Conservator.create(ctx_obj["config_name"])
# raises InvalidLocalDatasetPath if the path does not point to a
# valid LocalDataset (defined as a directory containing index.json).
local_dataset = LocalDataset(conservator, path)
return func(local_dataset, *args, **kwargs)
def download_datasets(self, path, no_meter=False):
"""Clones and pulls all datasets in the collection."""
fields = FieldsRequest()
fields.include_field("name", "repository.master")
datasets = self.get_datasets(fields=fields)
for dataset in datasets:
clone_path = os.path.join(path, dataset.name)
lds = LocalDataset.clone(dataset, clone_path=clone_path)
lds.download(no_meter=no_meter)
def test_add_commit_push_frame(conservator, tmp_cwd, test_data):
local_path = test_data / "jpg" / "banana_0.jpg"
dataset = conservator.datasets.create("My Dataset")
local_dset = LocalDataset.clone(dataset)
local_dset.stage_local_images([local_path])
staged_paths = [
pathlib.Path(path) for path in local_dset.get_staged_images()
]
assert local_path in staged_paths and len(staged_paths) == 1
local_dset.push_staged_images()
assert os.path.exists(local_dset.frames_path)
frames = local_dset.get_frames()
assert len(frames) == 1
assert len(local_dset.get_staged_images()) == 0
local_dset.add_local_changes()
status = local_dset.git_status()
assert os.path.basename(
local_dset.frames_path) in status["added"]["staged"]
local_dset.commit("Test commit", verbose=False)
status = local_dset.git_status()
for category in ("added", "modified"):
for section in ("staged", "working"):
assert not status[category][section]
assert not status["other"]
local_dset.push_commits(verbose=False)
max_tries = 10
while not os.path.exists(local_dset.frames_path):
local_dset.pull(verbose=False)
time.sleep(1)
max_tries -= 1
if max_tries <= 0:
break
assert os.path.exists(local_dset.frames_path)
def test_clone(conservator, tmp_cwd):
dataset = conservator.datasets.create("My Dataset")
local_dset = LocalDataset.clone(dataset)
assert os.path.exists("My Dataset")
assert os.path.exists("My Dataset/index.json")
assert os.path.exists("My Dataset/dataset.jsonl")
import logging
from FLIR.conservator.conservator import Conservator
from FLIR.conservator.local_dataset import LocalDataset
logging.basicConfig(level=logging.INFO)
conservator = Conservator.default()
# Cloning from a known ID:
known_dataset = conservator.datasets.from_id("wJQfFkgNDpTqTQGv3")
known_dataset_local = LocalDataset.clone(known_dataset)
# Cloning can of course be done from any instance--as long as its
# ID is known. Cloning the first from a search:
deer_dataset = conservator.datasets.search("deer").with_fields("id").first()
deer_dataset_local = LocalDataset.clone(deer_dataset)
def clone(ctx, identifier, path, checkout):
conservator = Conservator.create(ctx.obj["config_name"])
dataset = conservator.datasets.from_string(identifier)
cloned = LocalDataset.clone(dataset, path)
if checkout is not None:
cloned.checkout(checkout)