def test_read_template_manifest():
"""Test reading template manifest file.
It creates a fake manifest file and it verifies it's read property.
"""
with TemporaryDirectory() as tempdir:
template_file = Path(tempdir) / TEMPLATE_MANIFEST
# error on missing template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.touch(exist_ok=True)
# error on invalid template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text("-\n"
" folder: first\n"
" name: Basic Project 1\n"
" description: Description 1\n"
" variables: {}\n"
"-\n"
" folder: second\n"
" name: Basic Project 2\n"
" description: Description 2\n"
" variables:\n"
" custom: Custom Value\n")
manifest = read_template_manifest(Path(tempdir), checkout=False)
assert 2 == len(manifest)
assert "first" == manifest[0]["folder"]
assert "second" == manifest[1]["folder"]
assert "Basic Project 1" == manifest[0]["name"]
assert "Description 2" == manifest[1]["description"]
variables1 = manifest[0]["variables"]
variables2 = manifest[1]["variables"]
assert 0 == len(variables1)
assert 1 == len(variables2)
assert 1 == len(variables2.keys())
assert "custom" in variables2.keys()
assert "Custom Value" == variables2["custom"]
template_file.write_text("-\n"
" folder: first\n"
" description: Description 1\n")
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text("-\n"
" name: Basic Project 2\n"
" description: Description 2\n")
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
def test_read_template_manifest():
"""Test reading template manifest file.
It creates a fake manifest file and it verifies it's read property.
"""
with TemporaryDirectory() as tempdir:
template_file = Path(tempdir) / TEMPLATE_MANIFEST
# error on missing template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.touch(exist_ok=True)
# error on invalid template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text('-\n'
' folder: first\n'
' name: Basic Project 1\n'
' description: Description 1\n'
' variables: {}\n'
'-\n'
' folder: second\n'
' name: Basic Project 2\n'
' description: Description 2\n'
' variables:\n'
' custom: Custom Value\n')
manifest = read_template_manifest(Path(tempdir), checkout=False)
assert 2 == len(manifest)
assert 'first' == manifest[0]['folder']
assert 'second' == manifest[1]['folder']
assert 'Basic Project 1' == manifest[0]['name']
assert 'Description 2' == manifest[1]['description']
variables1 = manifest[0]['variables']
variables2 = manifest[1]['variables']
assert 0 == len(variables1)
assert 1 == len(variables2)
assert 1 == len(variables2.keys())
assert 'custom' in variables2.keys()
assert 'Custom Value' == variables2['custom']
template_file.write_text('-\n'
' folder: first\n'
' description: Description 1\n')
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text('-\n'
' name: Basic Project 2\n'
' description: Description 2\n')
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
def test_validate_template():
"""Test template validation.
It validates each local template.
"""
with TemporaryDirectory() as tempdir:
temppath = Path(tempdir)
# file error
with raises(errors.InvalidTemplateError):
validate_template(temppath)
# folder error
shutil.rmtree(str(tempdir))
renku_dir = temppath / RENKU_HOME
renku_dir.mkdir(parents=True)
with raises(errors.InvalidTemplateError):
validate_template(temppath)
# valid template
shutil.rmtree(str(tempdir))
shutil.copytree(str(template_local), str(tempdir))
manifest = read_template_manifest(Path(tempdir))
for template in manifest:
template_folder = temppath / template['folder']
assert validate_template(template_folder) is True
def test_create_from_template(local_client):
"""Test repository creation from a template.
It creates a renku projects from one of the local templates and it verifies
the data are properly copied to the new renku project folder.
"""
with TemporaryDirectory() as tempdir:
temppath = Path(tempdir) / 'local'
shutil.copytree(str(template_local), str(temppath))
manifest = read_template_manifest(temppath)
template_path = temppath / manifest[0]['folder']
create_from_template(
template_path, local_client, METADATA['name'],
METADATA['description']
)
template_files = [
f
for f in local_client.path.glob('**/*') if '.git' not in str(f) and
not str(f).endswith('.renku/metadata.yml')
]
for template_file in template_files:
expected_file = template_path / template_file.relative_to(
local_client.path
)
assert expected_file.exists()
def test_compare_manifests(svc_client_with_templates):
"""Check reading manifest template."""
svc_client, headers, template_params = svc_client_with_templates
response = svc_client.get("/templates.read_manifest",
query_string=template_params,
headers=headers)
assert response
assert {"result"} == set(response.json.keys())
assert response.json["result"]["templates"]
with TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
manifest_file = fetch_template(template_params["url"],
template_params["ref"], temp_path)
manifest = read_template_manifest(temp_path)
assert manifest_file and manifest_file.exists()
assert manifest
templates_service = response.json["result"]["templates"]
templates_local = manifest
default_index = template_params["index"] - 1
assert templates_service[default_index] == templates_local[
default_index]
def test_compare_manifests(svc_client_with_templates):
"""Check reading manifest template."""
svc_client, headers, template_params = svc_client_with_templates
response = svc_client.get('/templates.read_manifest',
query_string=template_params,
headers=headers)
assert response
assert {'result'} == set(response.json.keys())
assert response.json['result']['templates']
with TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
manifest_file = fetch_template(TEMPLATE_URL, TEMPLATE_REF, temp_path)
manifest = read_template_manifest(temp_path)
assert manifest_file and manifest_file.exists()
assert manifest
templates_service = response.json['result']['templates']
templates_local = manifest
default_index = TEMPLATE_INDEX - 1
assert templates_service[default_index] == templates_local[
default_index]
def run(self):
from renku.core.commands.init import fetch_template, read_template_manifest
with TemporaryDirectory() as tempdir:
# download and extract template data
temppath = Path(tempdir)
print("downloading Renku templates...")
fetch_template(URL, REFERENCE, temppath)
read_template_manifest(temppath, checkout=True)
# copy templates
current_path = Path.cwd()
template_path = current_path / "renku" / "templates"
if template_path.exists():
shutil.rmtree(str(template_path))
shutil.copytree(str(temppath),
str(template_path),
ignore=shutil.ignore_patterns(".git"))
def test_read_template_manifest():
"""Test reading template manifest file.
It creates a fake manifest file and it verifies it's read propery.
"""
with TemporaryDirectory() as tempdir:
template_file = Path(tempdir) / TEMPLATE_MANIFEST
# error on missing template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.touch(exist_ok=True)
# error on invalid template file
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text('-\n'
' folder: first\n'
' name: Basic Project 1\n'
' description: Description 1\n'
'-\n'
' folder: second\n'
' name: Basic Project 2\n'
' description: Description 2\n')
manifest = read_template_manifest(Path(tempdir), checkout=False)
assert len(manifest) == 2
assert manifest[0]['folder'] == 'first'
assert manifest[1]['folder'] == 'second'
assert manifest[0]['name'] == 'Basic Project 1'
assert manifest[1]['description'] == 'Description 2'
template_file.write_text('-\n'
' folder: first\n'
' description: Description 1\n')
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
template_file.write_text('-\n'
' name: Basic Project 2\n'
' description: Description 2\n')
with raises(errors.InvalidTemplateError):
manifest = read_template_manifest(Path(tempdir), checkout=False)
def read_manifest_from_template(user, cache):
"""Read templates from the manifest file of a template repository."""
project_data = ManifestTemplatesRequest().load({
**user,
**request.args,
},
unknown=EXCLUDE)
project = _project_clone(user, project_data)
manifest = read_template_manifest(project.abs_path)
return result_response(ManifestTemplatesResponseRPC(),
{"templates": manifest})
def test_fetch_template_and_read_manifest():
"""Test template fetch and manifest reading.
It fetches a local template, reads the manifest, checkouts the
template folders and verify they exist.
"""
with TemporaryDirectory() as tempdir:
template_path = Path(tempdir)
fetch_template(TEMPLATE_URL, TEMPLATE_REF, template_path)
manifest = read_template_manifest(template_path, checkout=True)
for template in manifest:
template_folder = template_path / template['folder']
assert template_folder.exists()
def test_create_from_template(local_client, template):
"""Test repository creation from a template.
It creates a renku projects from one of the local templates and it verifies
the data are properly copied to the new renku project folder.
"""
with TemporaryDirectory() as tempdir:
temppath = Path(tempdir) / "local"
shutil.copytree(str(template_local), str(temppath))
manifest = read_template_manifest(temppath)
template_path = temppath / manifest[0]["folder"]
create_from_template(template_path, local_client, "name",
template["metadata"])
template_files = [
f for f in local_client.path.glob("**/*") if ".git" not in str(f)
and not str(f).endswith(".renku/metadata.yml")
]
for template_file in template_files:
expected_file = template_path / template_file.relative_to(
local_client.path)
assert expected_file.exists()
def init(ctx, client, use_external_storage, path, name, template,
template_source, template_ref, template_variables, description,
print_manifest, force):
"""Initialize a project in PATH. Default is current path."""
# verify dirty path
if not is_path_empty(path) and not force and not print_manifest:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template:
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['name'] == template
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo('The template "{0}" is not available.'.format(template))
repeat = True
if print_manifest:
if template_data:
click.echo(create_template_sentence([template_data]))
else:
click.echo(create_template_sentence(template_manifest))
return
if not template or repeat:
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# set local path and storage
store_directory(path)
if not client.use_external_storage:
use_external_storage = False
ctx.obj = client = attr.evolve(client,
path=path,
use_external_storage=use_external_storage)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path, client, name, description,
template_variables, force)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(
ctx,
client,
external_storage_requested,
path,
name,
template_id,
template_index,
template_source,
template_ref,
metadata,
list_templates,
force,
describe,
data_dir,
):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
existing_paths = [
str(p.relative_to(path)) for p in Path(path).iterdir()
]
existing_paths.sort()
raise errors.InvalidFileOperation(
f'Folder "{str(path)}" is not empty and contains the following files/directories:'
+ "".join((f"\n\t{e}" for e in existing_paths)) +
"\nPlease add --force flag to transform it into a Renku repository."
)
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
"The user name and email are not configured. "
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
"\tgit config --global --add user.email "
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo("Fetching template from {0}@{1}... ".format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho("OK", fg="green")
else:
template_folder = Path(
pkg_resources.resource_filename("renku", "templates"))
template_manifest = read_template_manifest(template_folder)
template_source = "renku"
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
"Use either --template-id or --template-index, not both",
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem["folder"] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f"The template at index {template_index} is not available.")
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_index = click.prompt(
text=create_template_sentence(templates,
describe=describe,
instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False,
)
template_data = templates[template_index - 1]
template_id = template_data["folder"]
# verify variables have been passed
template_variables = template_data.get("variables", {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(metadata.keys())
for key in template_variables_keys - input_parameters_keys:
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f"({template_variables[key]})"),
default="",
show_default=False,
)
metadata[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if len(useless_variables) > 0:
click.echo(INFO +
"These parameters are not used by the template and were "
"ignored:\n\t{}".format("\n\t".join(useless_variables)))
for key in useless_variables:
del metadata[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit("*")
branch_name = "pre_renku_init_{0}".format(commit.hexsha[:7])
with client.worktree(
path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
"--no-ff", "-s", "recursive", "-X", "ours",
"--allow-unrelated-histories"
],
):
click.echo(
"Saving current data in branch {0}".format(branch_name))
except AttributeError:
click.echo("Warning! Overwriting non-empty folder.")
except GitCommandError as e:
click.UsageError(e)
# supply additional metadata
metadata["__template_source__"] = template_source
metadata["__template_ref__"] = template_ref
metadata["__template_id__"] = template_id
metadata["__namespace__"] = ""
metadata["__sanitized_project_name__"] = ""
metadata["__repository__"] = ""
metadata["__project_slug__"] = ""
# clone the repo
template_path = template_folder / template_data["folder"]
click.echo("Initializing new Renku repository... ", nl=False)
with client.lock:
try:
create_from_template(
template_path=template_path,
client=client,
name=name,
metadata=metadata,
force=force,
data_dir=data_dir,
)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def init(ctx, client, external_storage_requested, path, name, template_id,
template_index, template_source, template_ref, parameter,
list_templates, force, describe, data_dir):
"""Initialize a project in PATH. Default is the current path."""
# verify dirty path
if not is_path_empty(path) and not force and not list_templates:
raise errors.InvalidFileOperation(
'Folder "{0}" is not empty. Please add --force '
'flag to transform it into a Renku repository.'.format(str(path)))
data_dir = resolve_data_directory(data_dir, path)
if not check_git_user_config():
raise errors.ConfigurationError(
'The user name and email are not configured. '
'Please use the "git config" command to configure them.\n\n'
'\tgit config --global --add user.name "John Doe"\n'
'\tgit config --global --add user.email '
'"*****@*****.**"\n')
# select template source
if template_source:
click.echo('Fetching template from {0}@{1}... '.format(
template_source, template_ref),
nl=False)
template_folder = Path(mkdtemp())
fetch_template(template_source, template_ref, template_folder)
template_manifest = read_template_manifest(template_folder,
checkout=True)
click.secho('OK', fg='green')
else:
template_folder = Path(
pkg_resources.resource_filename('renku', 'templates'))
template_manifest = read_template_manifest(template_folder)
# select specific template
repeat = False
template_data = None
if template_id:
if template_index:
raise errors.ParameterError(
'Use either --template-id or --template-index, not both',
'"--template-index"')
template_filtered = [
template_elem for template_elem in template_manifest
if template_elem['folder'] == template_id
]
if len(template_filtered) == 1:
template_data = template_filtered[0]
else:
click.echo(
f'The template with id "{template_id}" is not available.')
repeat = True
if template_index or template_index == 0:
if template_index > 0 and template_index <= len(template_manifest):
template_data = template_manifest[template_index - 1]
else:
click.echo(
f'The template at index {template_index} is not available.')
repeat = True
if list_templates:
if template_data:
click.echo(
create_template_sentence([template_data], describe=describe))
else:
click.echo(
create_template_sentence(template_manifest, describe=describe))
return
if repeat or not (template_id or template_index):
templates = [template_elem for template_elem in template_manifest]
if len(templates) == 1:
template_data = templates[0]
else:
template_num = click.prompt(text=create_template_sentence(
templates, describe=describe, instructions=True),
type=click.IntRange(1, len(templates)),
show_default=False,
show_choices=False)
template_data = templates[template_num - 1]
# verify variables have been passed
template_variables = template_data.get('variables', {})
template_variables_keys = set(template_variables.keys())
input_parameters_keys = set(parameter.keys())
for key in (template_variables_keys - input_parameters_keys):
value = click.prompt(
text=(f'The template requires a value for "{key}" '
f'({template_variables[key]})'),
default='',
show_default=False)
parameter[key] = value
useless_variables = input_parameters_keys - template_variables_keys
if (len(useless_variables) > 0):
click.echo(INFO +
'These parameters are not used by the template and were '
'ignored:\n\t{}'.format('\n\t'.join(useless_variables)))
for key in useless_variables:
del parameter[key]
# set local path and storage
store_directory(path)
if not client.external_storage_requested:
external_storage_requested = False
ctx.obj = client = attr.evolve(
client,
path=path,
data_dir=data_dir,
external_storage_requested=external_storage_requested)
if not is_path_empty(path):
from git import GitCommandError
try:
commit = client.find_previous_commit('*')
branch_name = 'pre_renku_init_{0}'.format(commit.hexsha[:7])
with client.worktree(path=path,
branch_name=branch_name,
commit=commit,
merge_args=[
'--no-ff', '-s', 'recursive', '-X',
'ours', '--allow-unrelated-histories'
]):
click.echo(
'Saving current data in branch {0}'.format(branch_name))
except AttributeError:
click.echo('Warning! Overwriting non-empty folder.')
except GitCommandError as e:
click.UsageError(e)
# clone the repo
template_path = template_folder / template_data['folder']
click.echo('Initializing new Renku repository... ', nl=False)
with client.lock:
try:
create_from_template(template_path=template_path,
client=client,
name=name,
metadata=parameter,
force=force,
data_dir=data_dir)
except FileExistsError as e:
raise click.UsageError(e)
# Install git hooks
from .githooks import install
ctx.invoke(install, force=force)
def create_project_from_template(user, cache):
"""Create a new project starting form target template."""
ctx = ProjectTemplateRequest().load({
**user,
**request.json,
},
unknown=EXCLUDE)
# Clone project and find target template
template_project = _project_clone(user, ctx)
templates = read_template_manifest(template_project.abs_path)
template = next(
(template
for template in templates if template["folder"] == ctx["identifier"]),
None)
if template is None:
return error_response(INVALID_PARAMS_ERROR_CODE,
"invalid identifier for target repository")
# Verify missing parameters
template_parameters = template.get("variables", {})
provided_parameters = {p["key"]: p["value"] for p in ctx["parameters"]}
missing_keys = list(template_parameters.keys() -
provided_parameters.keys())
if len(missing_keys) > 0:
return error_response(INVALID_PARAMS_ERROR_CODE,
f"missing parameter: {missing_keys[0]}")
# Create new path
new_project_path = make_new_project_path(user, ctx)
if new_project_path.exists():
shutil.rmtree(str(new_project_path))
new_project_path.mkdir(parents=True, exist_ok=True)
default_metadata = {
"__template_source__": ctx["git_url"],
"__template_ref__": ctx["ref"],
"__template_id__": ctx["identifier"],
"__namespace__": ctx["project_namespace"],
"__repository__": ctx["project_repository"],
"__sanitized_project_name__": ctx["project_name_stripped"],
"__project_slug__": ctx["project_slug"],
}
# prepare data and init new project
source_path = template_project.abs_path / ctx["identifier"]
git_user = {"email": user["email"], "name": user["fullname"]}
with chdir(new_project_path):
create_from_template_local(
source_path,
ctx["project_name"],
provided_parameters,
default_metadata,
git_user,
ctx["url"],
ctx["ref"],
"service",
)
new_repo_push(new_project_path, ctx["new_project_url_with_auth"])
resp = {
"url": ctx["new_project_url"],
"namespace": ctx["project_namespace"],
"name": ctx["project_name_stripped"],
}
return result_response(ProjectTemplateResponseRPC(), resp)
