def add_file(self, path, revision='HEAD'):
"""Add a file node to the graph."""
file_commits = list(self.client.git.iter_commits(revision, paths=path))
if not file_commits:
raise KeyError('Could not find a file {0} in range {1}'.format(
path, revision))
commit = file_commits[0]
cwl = self.find_cwl(commit)
if cwl is not None:
file_key = self.add_node(commit, path)
self.add_tool(commit, cwl, file_key=file_key)
return file_key
else:
#: Does not have a parent CWL.
root_node = self.add_node(commit, path)
parent_commit, parent_path = root_node
#: Capture information about the submodule in a submodule.
root_submodule = self.G.nodes[root_node].get('submodule', [])
#: Resolve Renku based submodules.
original_path = Path(parent_path)
if original_path.is_symlink() or str(original_path).startswith(
'.renku/vendors'):
original_path = original_path.resolve()
for submodule in Submodule.iter_items(
self.client.git, parent_commit=parent_commit):
try:
subpath = original_path.relative_to(
Path(submodule.path).resolve())
subgraph = Graph(client=LocalClient(
path=submodule.path))
subnode = subgraph.add_file(str(subpath),
revision=submodule.hexsha)
#: Extend node metadata.
for _, data in subgraph.G.nodes(data=True):
data['submodule'] = root_submodule + [
submodule.name
]
#: Merge file node with it's symlinked version.
self.G = nx.contracted_nodes(
nx.compose(self.G, subgraph.G),
root_node,
subnode,
) # TODO optionally it can be changed to an edge.
break
except ValueError:
continue
return root_node
def move(ctx, client, sources, destination):
"""Move files and check repository for potential problems."""
from renku.api._git import _expand_directories
dst = Path(destination)
def fmt_path(path):
"""Format path as relative to the client path."""
return str(Path(path).absolute().relative_to(client.path))
files = {
fmt_path(source): fmt_path(file_or_dir)
for file_or_dir in sources
for source in _expand_directories((file_or_dir, ))
}
def fmt_dst(path):
"""Build a destination path for a source path."""
return str(dst / os.path.relpath(path, start=files[path]))
destinations = {source: fmt_dst(source) for source in files}
# 1. Check .gitignore.
ignored = client.find_ignored_paths(*destinations.values())
if ignored:
click.echo(WARNING + 'Renamed files match .gitignore.\n')
if click.confirm('Do you want to edit ".gitignore" now?',
default=False):
click.edit(filename=str(client.path / '.gitignore'))
# 2. Update dataset metadata files.
with progressbar(
client.datasets.items(),
item_show_func=lambda item: str(item[1].short_id) if item else '',
label='Updating dataset metadata',
width=0,
) as bar:
for (path, dataset) in bar:
renames = {}
for file_ in dataset.files:
filepath = fmt_path(file_.path)
if filepath in files:
renames[file_.path] = destinations[filepath]
if renames:
dataset = dataset.rename_files(
lambda key: renames.get(key, key))
dataset.to_yaml()
# 3. Manage .gitattributes for external storage.
tracked = tuple()
if client.has_external_storage:
tracked = tuple(path
for path, attr in client.find_attr(*files).items()
if attr.get('filter') == 'lfs')
client.untrack_paths_from_storage(*tracked)
if client.find_attr(*tracked):
click.echo(WARNING + 'There are custom .gitattributes.\n')
if click.confirm('Do you want to edit ".gitattributes" now?',
default=False):
click.edit(filename=str(client.path / '.gitattributes'))
if tracked and client.has_external_storage:
client.track_paths_in_storage(*(destinations[path]
for path in tracked))
# 4. Handle symlinks.
dst.parent.mkdir(parents=True, exist_ok=True)
for source, target in destinations.items():
src = Path(source)
if src.is_symlink():
Path(target).parent.mkdir(parents=True, exist_ok=True)
Path(target).symlink_to(
os.path.relpath(str(src.resolve()),
start=os.path.dirname(target)))
src.unlink()
del files[source]
# Finally move the files.
final_sources = list(set(files.values()))
if final_sources:
run(['git', 'mv'] + final_sources + [destination], check=True)
