def freeze(slug):
# Validate the input
try:
dtype, name, tag = FreezableAPI.parse_slug(slug)
if tag is None:
raise TagInvalidError()
except (TagInvalidError, FreezableNameInvalidError):
click.echo(f'Please provide a valid tag in "{slug}"')
sys.exit(1)
# Make sure that the dataset is available
if not FreezableAPI.exists(dtype, name):
click.echo(f'"{dtype}.{name}" not in minus80 datasets! '
"check available datasets with the ls command")
sys.exit(1)
else:
# Create the minus80
try:
dataset = getattr(m80, dtype)(name)
except Exception as e: # pragma: no cover
click.echo(f"Could not build {dtype}.{name}")
raise e
sys.exit(1)
# Freeze with tag
try:
dataset.m80.freeze(tag)
click.echo(click.style("SUCCESS!", fg="green", bold=True))
sys.exit(0)
except TagExistsError:
click.echo(
f'Tag "{tag}" already exists in the cloud for {dtype}.{name}')
sys.exit(1)
def thaw(slug, force):
try:
cwd = Path.cwd().resolve()
except FileNotFoundError: # pragma: no cover
cwd = "/"
try:
dtype, name, tag = FreezableAPI.parse_slug(slug)
if tag is None:
raise TagInvalidError()
except (TagInvalidError, FreezableNameInvalidError):
click.echo(f'Please provide a valid tag in "{slug}"')
sys.exit(1)
# Make sure that the dataset is available
if not FreezableAPI.exists(dtype, name):
click.echo(f'"{dtype}.{name}" not in minus80 datasets! '
"check available datasets with the ls command")
sys.exit(1)
else:
# Create the minus80
try:
dataset = getattr(m80, dtype)(name)
except Exception: # pragma: no cover
click.echo(f"Could not build {dtype}.{name}")
# Freeze with tag
try:
dataset.m80.thaw(tag, force=force)
click.echo(click.style("SUCCESS!", fg="green", bold=True))
sys.exit(0)
except TagDoesNotExistError:
click.echo(f'tag "{tag}" does not exist for {dtype}.{name}')
sys.exit(1)
except UnsavedChangesInThawedError as e:
click.secho(
'freeze your current changes or use "force" to dispose of '
"any unsaved changes in current thawed dataset",
fg="red",
)
for status, files in {
"Changed": e.changed,
"New": e.new,
"Deleted": e.deleted,
}.items():
for f in files:
click.secho(f" {status}: {f}", fg="yellow")
sys.exit(1)
# Warn the user if they are in a directory (cwd) that was deleted
# in the thaw -- theres nothing we can do about this ...
if str(cwd).startswith(str(dataset.m80.thawed_dir)):
click.echo(
"Looks like you are currently in a directory that was just thawed, "
"update your current working directory with, e.g.:\n"
"$ cd `pwd`\n"
f"$ cd {cwd}")
def pull(slug):
"""
\b
Pull a frozen minus80 dataset from the cloud.
\b
Positional Arguments:
<slug> - The slug of the frozen minus80 dataset (e.g. Project.foo:v1)
"""
cloud = m80.CloudData()
try:
cloud.user
except UserNotLoggedInError:
click.secho("Please log in to use this feature")
return 1
try:
dtype, name, tag = FreezableAPI.parse_slug(slug)
if tag is None:
raise TagInvalidError()
except (TagInvalidError, FreezableNameInvalidError):
click.echo(f'Please provide a valid tag in "{slug}"')
return 1
# Pull the files and tag from the cloud
try:
# run the push method in an event loop
asyncio.run(cloud.pull(dtype, name, tag))
except TagExistsError:
click.echo(f"The tag ({tag}) already exists from {dtype}.{name}")
return 1
except CloudDatasetDoesNotExistError:
click.echo(f'The dataset "{dtype}.{name}" does not exist in the cloud')
return 1
except CloudTagDoesNotExistError:
click.echo(f'The tag "{tag}" does not exist in the cloud')
return 1
except CloudPullFailedError:
click.echo(f'Failed to pull all data for tag "{tag}". ')
click.echo("This could be network issues, please try again later ")
click.echo(
"or report error to https://github.com/LinkageIO/minus80/issues ")
return 1
# Let the user know
click.echo(f"{dtype}.{name}:{tag} successfully pulled")
def push(slug):
"""
\b
Push a frozen minus80 dataset to the cloud.
\b
Positional Arguments:
<slug> - A slug of a frozen minus80 dataset
"""
cloud = m80.CloudData()
try:
cloud.user
except UserNotLoggedInError:
click.secho("Please log in to use this feature")
try:
dtype, name, tag = FreezableAPI.parse_slug(slug)
if tag is None:
raise TagInvalidError()
except (TagInvalidError, FreezableNameInvalidError):
click.echo(f'Please provide a valid tag in "{slug}"')
return 0
# Make sure that the dataset is available
if not FreezableAPI.exists(dtype, name):
click.echo(f'"{dtype}.{name}" not in minus80 datasets! '
"check available datasets with the ls command")
return 0
else:
try:
# run the push method in an event loop
asyncio.run(cloud.push(dtype, name, tag))
except TagDoesNotExistError:
click.echo(f'tag "{tag}" does not exist for {dtype}.{name}')
except TagExistsError:
click.echo(f"Cannot push {dtype}.{name}:{tag} to the cloud.")
click.echo("The tag already exists there.")
except TagConflictError:
click.echo(f"Cannot push {dtype}.{name}:{tag} to the cloud.")
click.echo("The tag already exists there.")
click.secho(
"Warning! The contents of the local tag and the cloud tag differ "
"Create a tag with a unique name and retry pushing",
fg="red",
)
def ls(name, dtype, tags):
if dtype is None:
dtype = "*"
if name is None:
name = "*"
files = FreezableAPI.datasets(dtype=dtype, name=name)
# Print message if nothing is here
if len(files) == 0: # pragma: no cover
print("[Nothing here yet]")
return None
# group by dtype and print
datasets = defaultdict(list)
for slug in files:
dtype, name, tag = FreezableAPI.parse_slug(slug)
datasets[dtype].append(name)
# Print a formatted table
for dtype, names in datasets.items():
print(f"{dtype}")
for i, name in enumerate(names, 1):
self = FreezableAPI(dtype, name)
print(f" └──{self.name}")
# Print tag data
if tags:
thawed_tag = None
tags = []
for t in self.tag_data:
if t["tag"] == "thawed":
# TODO: add thawed info into ls
thawed_tag = t
assert thawed_tag
else:
tags.append(t)
tags.sort(key=lambda x: x["timestamp"])
# print thawed info first
for t in tags:
timestamp = datetime.fromtimestamp(
t["timestamp"]).strftime("%I:%M%p - %b %d, %Y")
csum = t["total"][0:10]
print(f" └──{t['tag']} {csum} ({timestamp})")
sys.exit(0)
def delete(slug, force):
# Validate the input
if force is False: # pragma: no cover
click.confirm(f'Are you sure you want to delete "{slug}"')
try:
dtype, name, tag = FreezableAPI.parse_slug(slug)
if tag is not None:
raise TagInvalidError()
except TagInvalidError:
click.echo("Cannot delete tags, only entire datasets.")
sys.exit(1)
except FreezableNameInvalidError:
click.echo(
"Please provide a valid dataset name: <dtype>.<name>. E.g. Project.foobar"
"Note: do not include tags")
sys.exit(1)
# Make sure that the dataset is available
if not FreezableAPI.exists(dtype, name):
click.echo(f'"{dtype}.{name}" not in minus80 datasets! '
"check available datasets with the ls command")
sys.exit(1)
else:
FreezableAPI.delete(dtype, name)
sys.exit(0)
def test_bad_freezable_name():
with pytest.raises(FreezableNameInvalidError):
FreezableAPI.parse_slug("Project/foobar")
def test_parse_slug_no_tag():
assert FreezableAPI.parse_slug("Project.foobar") == ("Project", "foobar",
None)
def test_parse_slug():
assert FreezableAPI.parse_slug("Project.foobar:v1") == ("Project",
"foobar", "v1")