def neglect_submodule_templates(project_root, template_list):
template_dict = {}
# one template object per path.
for template in template_list:
template_dict[template.template_path] = template
for template_descendent in template.descendents:
template_dict[
template_descendent.template_path] = template_descendent
# Removing those within a submodule.
submodule_path_prefixes = []
try:
gitmodule_config = ConfigFile.from_path(
Path(project_root / ".gitmodules"))
except FileNotFoundError:
return template_list
for submodule_path, _, _ in parse_submodules(gitmodule_config):
submodule_path_prefixes.append(
Path(project_root / submodule_path.decode("utf-8")))
finalized_templates = []
for template_obj in list(template_dict.values()):
gitmodule_template = False
for gm_path in submodule_path_prefixes:
if gm_path in template_obj.template_path.parents:
gitmodule_template = True
if not gitmodule_template:
finalized_templates.append(template_obj)
return finalized_templates
def _clone_submodules(cls, repo: Repo) -> None:
"""
Helper method to identify configured submodules and clone them recursively.
"""
repo_root = Path(repo.path)
modules_config = repo_root.joinpath(".gitmodules")
if modules_config.exists():
config = ConfigFile.from_path(modules_config)
url: bytes
path: bytes
submodules = parse_submodules(config) # type: ignore[no-untyped-call]
for path, url, _ in submodules:
path_relative = Path(path.decode("utf-8"))
path_absolute = repo_root.joinpath(path_relative)
source_root = path_absolute.parent
source_root.mkdir(parents=True, exist_ok=True)
with repo:
revision = repo.open_index()[path].sha.decode("utf-8")
cls.clone(
url=url.decode("utf-8"),
source_root=source_root,
name=path_relative.name,
revision=revision,
clean=path_absolute.exists()
and not path_absolute.joinpath(".git").is_dir(),
)
def fetch_refs(remote_name = 'origin', local='.'):
"""
Fetch references from a Git remote repository
:param remote_name: <str> git name of remote repository, _default='origin'_
:param local: <str> full path to local repository, _default='.'_
:return entries: <TreeEntry> named tuples
"""
#import rpdb; rpdb.set_trace()
# **Fetch refs from remote**
# create a dulwich Repo object from path to local repo
r = Repo(local) # local repository
objsto = r.object_store # create a ObjectStore object from the local repo
determine_wants = objsto.determine_wants_all # built in dulwich function
gitdir = os.path.join(local, r.controldir()) # the git folder
cnf_file = os.path.join(gitdir, 'config') # path to config
cnf = ConfigFile.from_path(cnf_file) # config
remote = cnf.get(('remote', remote_name), 'url') # url of remote
# correctly parse host path and create dulwich Client object from it
client, host_path = get_transport_and_path(remote)
remote_refs = client.fetch(host_path, r, determine_wants, sys.stdout.write)
# **Store refs fetched by dulwich**
dulwich_refs = os.path.join(gitdir, DULWICH_REFS)
with open(dulwich_refs, 'wb') as file:
writer = csv.writer(file, delimiter=' ')
for key, value in remote_refs.items():
writer.writerow([key, value])
# **save remote refs shas for future checkout**
remote_dir = os.path.join(gitdir, 'refs', 'remotes', remote_name) # .git/refs/remotes
ensure_dir_exists(remote_dir) # built in dulwich function
headref = 0
# head branch ref
if remote_refs.has_key('HEAD'):
headref = remote_refs.pop('HEAD') # sha of HEAD
i_head = remote_refs.values().index(headref) # index of head ref
head_branch = remote_refs.keys()[i_head] # name of head branch
branch_key = head_branch.rsplit('/',1)[-1] # branch
head_file = os.path.join(remote_dir, 'HEAD') # path to branch shas file
head_ref = '/'.join(['refs','remotes',remote_name,branch_key])
with open(head_file, 'wb') as GitFile:
GitFile.write('ref: ' + head_ref + '\n')
# remote branch refs
for key, value in remote_refs.items():
key = key.rsplit('/',1)[-1] # get just the remote's branch
reffile = os.path.join(remote_dir, key) # path to branch shas file
with open(reffile, 'wb') as GitFile:
GitFile.write(value + '\n')
if headref:
remote_refs['HEAD'] = headref # restore HEAD sha
return remote_refs
def get_config(self):
"""Retrieve the config object.
Returns: `ConfigFile` object for the ``.git/config`` file.
"""
from dulwich.config import ConfigFile
path = os.path.join(self._controldir, 'config')
try:
return ConfigFile.from_path(path)
except FileNotFoundError:
ret = ConfigFile()
ret.path = path
return ret
def _find_submodules(self) -> Dict[str, "PathInfo"]:
"""Return dict mapping submodule names to submodule paths.
Submodule paths will be relative to Git repo root.
"""
from dulwich.config import ConfigFile, parse_submodules
submodules: Dict[str, "PathInfo"] = {}
config_path = os.path.join(self.root_dir, ".gitmodules")
if os.path.isfile(config_path):
config = ConfigFile.from_path(config_path)
for path, _url, section in parse_submodules(config):
submodules[os.fsdecode(section)] = PathInfo(os.fsdecode(path))
return submodules
def get_config(self):
"""Retrieve the config object.
:return: `ConfigFile` object for the ``.git/config`` file.
"""
from dulwich.config import ConfigFile
path = os.path.join(self._controldir, 'config')
try:
return ConfigFile.from_path(path)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
ret = ConfigFile()
ret.path = path
return ret
def get_config(self):
"""Retrieve the config object.
:return: `ConfigFile` object for the ``.git/config`` file.
"""
from dulwich.config import ConfigFile
path = os.path.join(self._controldir, 'config')
try:
return ConfigFile.from_path(path)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
ret = ConfigFile()
ret.path = path
return ret
def _recurse_submodules(self, path: Path, parent_url):
gitmodule_path = path / ".gitmodules"
if not gitmodule_path.is_file():
return
conf = ConfigFile.from_path(str(gitmodule_path))
for sub_path, url, name in parse_submodules(conf):
sub_path = sub_path.decode("utf-8")
url = url.decode("utf-8")
name = name.decode("utf-8")
if not (path / sub_path).is_dir():
(path / sub_path).mkdir(parents=True)
# bizarre process here, but I don't know how else to get the sha for the
# submodule...
sha = None
try:
porcelain.get_object_by_path(str(path), sub_path)
except KeyError as e:
sha = e.args[0].decode("utf-8")
if not sha:
raise ValueError(f"Could not find sha for submodule {name}")
if url.startswith("../"):
base_url = parent_url
for _ in range(url.count("../")):
base_url = "/".join(base_url.split("/")[:-1])
url = base_url + "/" + url.replace("../", "")
outp = BytesIO()
if not (path / sub_path / ".git").is_dir():
LOG.info(f"fetching git submodule {url}")
porcelain.clone(
url,
str(path / sub_path),
checkout=sha,
errstream=outp,
outstream=outp,
)
LOG.debug(outp.getvalue().decode("utf-8"))
self._recurse_submodules((path / sub_path), url)
def gen_configs():
for path in config_file + self._config_files:
try:
yield ConfigFile.from_path(path)
except (IOError, OSError, ValueError):
continue
def gen_configs():
for path in config_file + self._config_files:
try:
yield ConfigFile.from_path(path)
except (IOError, OSError, ValueError):
continue
