def test_fetch_template_and_read_manifest():
"""Test template fetch and manifest reading.
It fetches a local template, reads the manifest, checkouts the
template folders and verify they exist.
"""
with TemporaryDirectory() as tempdir:
template_path = Path(tempdir)
fetch_template(TEMPLATE_URL, TEMPLATE_REF, template_path)
manifest = read_template_manifest(template_path, checkout=True)
for template in manifest:
template_folder = template_path / template['folder']
assert template_folder.exists()
def test_compare_manifests(svc_client_with_templates):
"""Check reading manifest template."""
svc_client, headers, template_params = svc_client_with_templates
response = svc_client.get("/templates.read_manifest",
query_string=template_params,
headers=headers)
assert response
assert {"result"} == set(response.json.keys())
assert response.json["result"]["templates"]
with TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
manifest_file = fetch_template(template_params["url"],
template_params["ref"], temp_path)
manifest = read_template_manifest(temp_path)
assert manifest_file and manifest_file.exists()
assert manifest
templates_service = response.json["result"]["templates"]
templates_local = manifest
default_index = template_params["index"] - 1
assert templates_service[default_index] == templates_local[
default_index]
def test_compare_manifests(svc_client_with_templates):
"""Check reading manifest template."""
svc_client, headers, template_params = svc_client_with_templates
response = svc_client.get('/templates.read_manifest',
query_string=template_params,
headers=headers)
assert response
assert {'result'} == set(response.json.keys())
assert response.json['result']['templates']
with TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
manifest_file = fetch_template(TEMPLATE_URL, TEMPLATE_REF, temp_path)
manifest = read_template_manifest(temp_path)
assert manifest_file and manifest_file.exists()
assert manifest
templates_service = response.json['result']['templates']
templates_local = manifest
default_index = TEMPLATE_INDEX - 1
assert templates_service[default_index] == templates_local[
default_index]
def test_fetch_template(template):
"""Test fetching a template.
It fetches a template from remote and verifies that the manifest
file is there.
"""
# ? Can't parametrize with fixtures yet
# ? REF: https://docs.pytest.org/en/stable/proposals/parametrize_with_fixtures.html
parametrize = [
{
"url": template["url"],
"ref": template["ref"],
"error": None
},
{
"url": "fake",
"ref": template["ref"],
"error": errors.GitError
},
{
"url": template["url"],
"ref": "fake",
"error": errors.GitError
},
]
for parameters in parametrize:
with TemporaryDirectory() as tempdir:
with raises(parameters["error"]):
manifest_file = fetch_template(parameters["url"],
parameters["ref"],
Path(tempdir))
assert Path(tempdir) / TEMPLATE_MANIFEST == manifest_file
assert manifest_file.exists()
def run(self):
from renku.core.commands.init import fetch_template, read_template_manifest
with TemporaryDirectory() as tempdir:
# download and extract template data
temppath = Path(tempdir)
print("downloading Renku templates...")
fetch_template(URL, REFERENCE, temppath)
read_template_manifest(temppath, checkout=True)
# copy templates
current_path = Path.cwd()
template_path = current_path / "renku" / "templates"
if template_path.exists():
shutil.rmtree(str(template_path))
shutil.copytree(str(temppath),
str(template_path),
ignore=shutil.ignore_patterns(".git"))
def test_fetch_template(url, ref, result, error):
"""Test fetching a template.
It fetches a template from remote and verifies that the manifest
file is there.
"""
with TemporaryDirectory() as tempdir:
with raises(error):
manifest_file = fetch_template(url, ref, Path(tempdir))
assert manifest_file == Path(tempdir) / TEMPLATE_MANIFEST
assert manifest_file.exists()
def init(ctx, client, use_external_storage, path, name, template,
template_source, template_ref, template_variables, description,
print_manifest, force):
"""Initialize a project in PATH. Default is current path."""
# verify dirty path
if not is_path_empty(path) and not force and not print_manifest:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template:
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['name'] == template
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo('The template "{0}" is not available.'.format(template))
repeat = True
if print_manifest:
if template_data:
click.echo(create_template_sentence([template_data]))
else:
click.echo(create_template_sentence(template_manifest))
return
if not template or repeat:
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# set local path and storage
store_directory(path)
if not client.use_external_storage:
use_external_storage = False
ctx.obj = client = attr.evolve(client,
path=path,
use_external_storage=use_external_storage)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path, client, name, description,
template_variables, force)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(
ctx,
client,
external_storage_requested,
path,
name,
template_id,
template_index,
template_source,
template_ref,
metadata,
list_templates,
force,
describe,
data_dir,
):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
existing_paths = [
str(p.relative_to(path)) for p in Path(path).iterdir()
]
existing_paths.sort()
raise errors.InvalidFileOperation(
f'Folder "{str(path)}" is not empty and contains the following files/directories:'
+ "".join((f"\n\t{e}" for e in existing_paths)) +
"\nPlease add --force flag to transform it into a Renku repository."
)
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
"The user name and email are not configured. "
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
"\tgit config --global --add user.email "
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo("Fetching template from {0}@{1}... ".format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho("OK", fg="green")
else:
template_folder = Path(
pkg_resources.resource_filename("renku", "templates"))
template_manifest = read_template_manifest(template_folder)
template_source = "renku"
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
"Use either --template-id or --template-index, not both",
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem["folder"] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f"The template at index {template_index} is not available.")
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_index = click.prompt(
text=create_template_sentence(templates,
describe=describe,
instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False,
)
template_data = templates[template_index - 1]
template_id = template_data["folder"]
# verify variables have been passed
template_variables = template_data.get("variables", {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(metadata.keys())
for key in template_variables_keys - input_parameters_keys:
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f"({template_variables[key]})"),
default="",
show_default=False,
)
metadata[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if len(useless_variables) > 0:
click.echo(INFO +
"These parameters are not used by the template and were "
"ignored:\n\t{}".format("\n\t".join(useless_variables)))
for key in useless_variables:
del metadata[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit("*")
branch_name = "pre_renku_init_{0}".format(commit.hexsha[:7])
with client.worktree(
path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
"--no-ff", "-s", "recursive", "-X", "ours",
"--allow-unrelated-histories"
],
):
click.echo(
"Saving current data in branch {0}".format(branch_name))
except AttributeError:
click.echo("Warning! Overwriting non-empty folder.")
except GitCommandError as e:
click.UsageError(e)
# supply additional metadata
metadata["__template_source__"] = template_source
metadata["__template_ref__"] = template_ref
metadata["__template_id__"] = template_id
metadata["__namespace__"] = ""
metadata["__sanitized_project_name__"] = ""
metadata["__repository__"] = ""
metadata["__project_slug__"] = ""
# clone the repo
template_path = template_folder / template_data["folder"]
click.echo("Initializing new Renku repository... ", nl=False)
with client.lock:
try:
create_from_template(
template_path=template_path,
client=client,
name=name,
metadata=metadata,
force=force,
data_dir=data_dir,
)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(ctx, client, external_storage_requested, path, name, template_id,
template_index, template_source, template_ref, parameter,
list_templates, force, describe, data_dir):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
'Use either --template-id or --template-index, not both',
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['folder'] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f'The template at index {template_index} is not available.')
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, describe=describe, instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# verify variables have been passed
template_variables = template_data.get('variables', {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(parameter.keys())
for key in (template_variables_keys - input_parameters_keys):
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f'({template_variables[key]})'),
default='',
show_default=False)
parameter[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if (len(useless_variables) > 0):
click.echo(INFO +
'These parameters are not used by the template and were '
'ignored:\n\t{}'.format('\n\t'.join(useless_variables)))
for key in useless_variables:
del parameter[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path=template_path,
client=client,
name=name,
metadata=parameter,
force=force,
data_dir=data_dir)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)