def add_file(self, path, revision='HEAD'): """Add a file node to the graph.""" file_commits = list(self.client.git.iter_commits(revision, paths=path)) if not file_commits: raise KeyError('Could not find a file {0} in range {1}'.format( path, revision)) commit = file_commits[0] cwl = self.find_cwl(commit) if cwl is not None: file_key = self.add_node(commit, path) self.add_tool(commit, cwl, file_key=file_key) return file_key else: #: Does not have a parent CWL. root_node = self.add_node(commit, path) parent_commit, parent_path = root_node #: Capture information about the submodule in a submodule. root_submodule = self.G.nodes[root_node].get('submodule', []) #: Resolve Renku based submodules. original_path = Path(parent_path) if original_path.is_symlink() or str(original_path).startswith( '.renku/vendors'): original_path = original_path.resolve() for submodule in Submodule.iter_items( self.client.git, parent_commit=parent_commit): try: subpath = original_path.relative_to( Path(submodule.path).resolve()) subgraph = Graph(client=LocalClient( path=submodule.path)) subnode = subgraph.add_file(str(subpath), revision=submodule.hexsha) #: Extend node metadata. for _, data in subgraph.G.nodes(data=True): data['submodule'] = root_submodule + [ submodule.name ] #: Merge file node with it's symlinked version. self.G = nx.contracted_nodes( nx.compose(self.G, subgraph.G), root_node, subnode, ) # TODO optionally it can be changed to an edge. break except ValueError: continue return root_node
def move(ctx, client, sources, destination): """Move files and check repository for potential problems.""" from renku.api._git import _expand_directories dst = Path(destination) def fmt_path(path): """Format path as relative to the client path.""" return str(Path(path).absolute().relative_to(client.path)) files = { fmt_path(source): fmt_path(file_or_dir) for file_or_dir in sources for source in _expand_directories((file_or_dir, )) } def fmt_dst(path): """Build a destination path for a source path.""" return str(dst / os.path.relpath(path, start=files[path])) destinations = {source: fmt_dst(source) for source in files} # 1. Check .gitignore. ignored = client.find_ignored_paths(*destinations.values()) if ignored: click.echo(WARNING + 'Renamed files match .gitignore.\n') if click.confirm('Do you want to edit ".gitignore" now?', default=False): click.edit(filename=str(client.path / '.gitignore')) # 2. Update dataset metadata files. with progressbar( client.datasets.items(), item_show_func=lambda item: str(item[1].short_id) if item else '', label='Updating dataset metadata', width=0, ) as bar: for (path, dataset) in bar: renames = {} for file_ in dataset.files: filepath = fmt_path(file_.path) if filepath in files: renames[file_.path] = destinations[filepath] if renames: dataset = dataset.rename_files( lambda key: renames.get(key, key)) dataset.to_yaml() # 3. Manage .gitattributes for external storage. tracked = tuple() if client.has_external_storage: tracked = tuple(path for path, attr in client.find_attr(*files).items() if attr.get('filter') == 'lfs') client.untrack_paths_from_storage(*tracked) if client.find_attr(*tracked): click.echo(WARNING + 'There are custom .gitattributes.\n') if click.confirm('Do you want to edit ".gitattributes" now?', default=False): click.edit(filename=str(client.path / '.gitattributes')) if tracked and client.has_external_storage: client.track_paths_in_storage(*(destinations[path] for path in tracked)) # 4. Handle symlinks. dst.parent.mkdir(parents=True, exist_ok=True) for source, target in destinations.items(): src = Path(source) if src.is_symlink(): Path(target).parent.mkdir(parents=True, exist_ok=True) Path(target).symlink_to( os.path.relpath(str(src.resolve()), start=os.path.dirname(target))) src.unlink() del files[source] # Finally move the files. final_sources = list(set(files.values())) if final_sources: run(['git', 'mv'] + final_sources + [destination], check=True)