def list_unpushed_lfs_paths(self, client=None):
"""List paths tracked in lfs for a client."""
client = client or self
if (len(client.repo.remotes) < 1
or not client.repo.active_branch.tracking_branch()):
raise errors.ConfigurationError(
'No git remote is configured for {} branch {}.'.format(
client.path, client.repo.active_branch.name) +
'Cleaning the storage cache would lead to a loss of data as ' +
'it is not on a server. Please see ' +
'https://www.atlassian.com/git/tutorials/syncing for ' +
'information on how to sync with a remote.')
try:
status = check_output(self._CMD_STORAGE_STATUS,
cwd=client.path,
encoding='UTF-8')
except (KeyboardInterrupt, OSError) as e:
raise errors.ParameterError(
'Couldn\'t run \'git lfs\':\n{0}'.format(e))
files = status.split('Objects to be committed:')[0].splitlines()[2:]
files = [
client.path / f.rsplit('(', 1)[0].strip() for f in files
if f.strip()
]
return files
def detect_registry_url(client, auto_login=True):
"""Return a URL of the Docker registry."""
repo = client.repo
config = repo.config_reader()
# Find registry URL in .git/config
remote_url = None
try:
registry_url = config.get_value('renku', 'registry', None)
except NoSectionError:
registry_url = None
remote_branch = repo.head.reference.tracking_branch()
if remote_branch is not None:
remote_name = remote_branch.remote_name
config_section = 'renku "{remote_name}"'.format(
remote_name=remote_name)
try:
registry_url = config.get_value(config_section, 'registry',
registry_url)
except NoSectionError:
pass
remote_url = repo.remotes[remote_name].url
if registry_url:
# Look in [renku] and [renku "{remote_name}"] for registry_url key.
url = GitURL.parse(registry_url)
elif remote_url:
# Use URL based on remote configuration.
url = GitURL.parse(remote_url)
# Replace gitlab. with registry. unless running on gitlab.com.
hostname_parts = url.hostname.split('.')
if len(hostname_parts) > 2 and hostname_parts[0] == 'gitlab':
hostname_parts = hostname_parts[1:]
hostname = '.'.join(['registry'] + hostname_parts)
url = attr.evolve(url, hostname=hostname)
else:
raise errors.ConfigurationError(
'Configure renku.repository_url or Git remote.')
if auto_login and url.username and url.password:
try:
subprocess.run([
'docker',
'login',
url.hostname,
'-u',
url.username,
'--password-stdin',
],
check=True,
input=url.password.encode('utf-8'))
except subprocess.CalledProcessError:
raise errors.AuthenticationError(
'Check configuration of password or token in the registry URL')
return url
def parse(cls, href):
"""Derive basic informations."""
for regex in _REPOSITORY_URLS:
matches = re.search(regex, href)
if matches:
return cls(href=href, regex=regex, **matches.groupdict())
else:
raise errors.ConfigurationError(
'"{href} is not a valid Git remote.'.format(href=href))
def from_git(cls, git):
"""Create an instance from a Git repo."""
git_config = git.config_reader()
try:
name = git_config.get_value('user', 'name', None)
email = git_config.get_value('user', 'email', None)
except (configparser.NoOptionError,
configparser.NoSectionError): # pragma: no cover
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# Check the git configuration.
if not name: # pragma: no cover
raise errors.MissingUsername()
if not email: # pragma: no cover
raise errors.MissingEmail()
return cls(name=name, email=email)
def get_user_info(git):
"""Get Git repository's owner name and email."""
git_config = git.config_reader()
try:
name = git_config.get_value("user", "name", None)
email = git_config.get_value("user", "email", None)
except (configparser.NoOptionError,
configparser.NoSectionError): # pragma: no cover
raise errors.ConfigurationError(
"The user name and email are not configured. "
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
"\tgit config --global --add user.email "
'"*****@*****.**"\n')
# Check the git configuration.
if not name: # pragma: no cover
raise errors.MissingUsername()
if not email: # pragma: no cover
raise errors.MissingEmail()
return name, email
def repo_sync(repo, message=None, remote=None, paths=None):
"""Commit and push paths."""
origin = None
saved_paths = []
# get branch that's pushed
if repo.active_branch.tracking_branch():
pushed_branch = repo.active_branch.tracking_branch().name
else:
pushed_branch = repo.active_branch.name
if remote:
# get/setup supplied remote for pushing
if repo.remotes:
existing = next((r for r in repo.remotes if r.url == remote), None)
if not existing:
existing = next((r for r in repo.remotes if r.name == remote),
None)
origin = next((r for r in repo.remotes if r.name == "origin"),
None)
if existing:
origin = existing
elif origin:
pushed_branch = uuid4().hex
origin = repo.create_remote(pushed_branch, remote)
if not origin:
origin = repo.create_remote("origin", remote)
elif not repo.active_branch.tracking_branch():
# No remote set on branch, push to available remote if only a single
# one is available
if len(repo.remotes) == 1:
origin = repo.remotes[0]
else:
raise errors.ConfigurationError(
"No remote has been set up for the current branch")
else:
# get remote that's set up to track the local branch
origin = repo.remotes[repo.active_branch.tracking_branch().remote_name]
if paths:
# commit uncommitted changes
try:
repo.git.add(*paths)
saved_paths = [d.b_path for d in repo.index.diff("HEAD")]
if not message:
# Show saved files in message
max_len = 100
message = "Saved changes to: "
paths_with_lens = reduce(
lambda c, x: c + [(x, c[-1][1] + len(x))], saved_paths,
[(None, len(message))])[1:]
# limit first line to max_len characters
message += " ".join(p if l < max_len else "\n\t" + p
for p, l in paths_with_lens)
repo.index.commit(message)
except git.exc.GitCommandError as e:
raise errors.GitError("Cannot commit changes") from e
try:
# push local changes to remote branch
if origin.refs and repo.active_branch in origin.refs:
origin.fetch()
origin.pull(repo.active_branch)
origin.push(repo.active_branch)
except git.exc.GitCommandError as e:
if "protected branches" not in e.stderr:
raise errors.GitError("Cannot push changes") from e
# push to new remote branch if original one is protected
pushed_branch = uuid4().hex
origin = repo.create_remote(pushed_branch, remote)
origin.push(repo.active_branch)
return saved_paths, pushed_branch
def init(ctx, client, use_external_storage, path, name, template,
template_source, template_ref, template_variables, description,
print_manifest, force):
"""Initialize a project in PATH. Default is current path."""
# verify dirty path
if not is_path_empty(path) and not force and not print_manifest:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template:
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['name'] == template
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo('The template "{0}" is not available.'.format(template))
repeat = True
if print_manifest:
if template_data:
click.echo(create_template_sentence([template_data]))
else:
click.echo(create_template_sentence(template_manifest))
return
if not template or repeat:
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# set local path and storage
store_directory(path)
if not client.use_external_storage:
use_external_storage = False
ctx.obj = client = attr.evolve(client,
path=path,
use_external_storage=use_external_storage)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path, client, name, description,
template_variables, force)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(
ctx,
client,
external_storage_requested,
path,
name,
template_id,
template_index,
template_source,
template_ref,
metadata,
list_templates,
force,
describe,
data_dir,
):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
existing_paths = [
str(p.relative_to(path)) for p in Path(path).iterdir()
]
existing_paths.sort()
raise errors.InvalidFileOperation(
f'Folder "{str(path)}" is not empty and contains the following files/directories:'
+ "".join((f"\n\t{e}" for e in existing_paths)) +
"\nPlease add --force flag to transform it into a Renku repository."
)
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
"The user name and email are not configured. "
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
"\tgit config --global --add user.email "
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo("Fetching template from {0}@{1}... ".format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho("OK", fg="green")
else:
template_folder = Path(
pkg_resources.resource_filename("renku", "templates"))
template_manifest = read_template_manifest(template_folder)
template_source = "renku"
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
"Use either --template-id or --template-index, not both",
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem["folder"] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f"The template at index {template_index} is not available.")
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_index = click.prompt(
text=create_template_sentence(templates,
describe=describe,
instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False,
)
template_data = templates[template_index - 1]
template_id = template_data["folder"]
# verify variables have been passed
template_variables = template_data.get("variables", {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(metadata.keys())
for key in template_variables_keys - input_parameters_keys:
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f"({template_variables[key]})"),
default="",
show_default=False,
)
metadata[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if len(useless_variables) > 0:
click.echo(INFO +
"These parameters are not used by the template and were "
"ignored:\n\t{}".format("\n\t".join(useless_variables)))
for key in useless_variables:
del metadata[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit("*")
branch_name = "pre_renku_init_{0}".format(commit.hexsha[:7])
with client.worktree(
path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
"--no-ff", "-s", "recursive", "-X", "ours",
"--allow-unrelated-histories"
],
):
click.echo(
"Saving current data in branch {0}".format(branch_name))
except AttributeError:
click.echo("Warning! Overwriting non-empty folder.")
except GitCommandError as e:
click.UsageError(e)
# supply additional metadata
metadata["__template_source__"] = template_source
metadata["__template_ref__"] = template_ref
metadata["__template_id__"] = template_id
metadata["__namespace__"] = ""
metadata["__sanitized_project_name__"] = ""
metadata["__repository__"] = ""
metadata["__project_slug__"] = ""
# clone the repo
template_path = template_folder / template_data["folder"]
click.echo("Initializing new Renku repository... ", nl=False)
with client.lock:
try:
create_from_template(
template_path=template_path,
client=client,
name=name,
metadata=metadata,
force=force,
data_dir=data_dir,
)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(ctx, client, external_storage_requested, path, name, template_id,
template_index, template_source, template_ref, parameter,
list_templates, force, describe, data_dir):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
'Use either --template-id or --template-index, not both',
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['folder'] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f'The template at index {template_index} is not available.')
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, describe=describe, instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# verify variables have been passed
template_variables = template_data.get('variables', {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(parameter.keys())
for key in (template_variables_keys - input_parameters_keys):
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f'({template_variables[key]})'),
default='',
show_default=False)
parameter[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if (len(useless_variables) > 0):
click.echo(INFO +
'These parameters are not used by the template and were '
'ignored:\n\t{}'.format('\n\t'.join(useless_variables)))
for key in useless_variables:
del parameter[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path=template_path,
client=client,
name=name,
metadata=parameter,
force=force,
data_dir=data_dir)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)