def list(
self,
type=None,
paradigm=None,
parse=False,
):
p = self.folder
if type:
if not isinstance(type, str):
type = type.__name__.lower()
p = os.path.join(p, type)
if paradigm is not None:
p = os.path.join(p, 'paradigm' if paradigm else 'singular')
p1 = subprocess.Popen("find -path *.desc -print0".split(),
stdout=subprocess.PIPE,
cwd=p)
p2 = subprocess.Popen("xargs -0 cat".split(),
stdin=p1.stdout,
stdout=subprocess.PIPE,
cwd=p)
p3 = subprocess.Popen(["cut", "-f2", '-d', '"'],
stdin=p2.stdout,
stdout=subprocess.PIPE,
cwd=p)
p4 = subprocess.Popen(["uniq"],
stdin=p3.stdout,
stdout=subprocess.PIPE,
cwd=p)
res = [
s.strip().decode('utf8') for s in p4.stdout.readlines()
if s.strip()
]
if parse:
parser = IEMLParser(dictionary=self.get_dictionary())
_res = []
for s in res:
try:
_res.append(parser.parse(s))
except CannotParse as e:
error("Cannot parse {} : {}".format(s, repr(e)))
return _res
return res
def set_descriptors(self, ieml, descriptor, value):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
ieml = _check_ieml(ieml)
value = _check_descriptors(value)
desc = db.get_descriptors()
old_trans = {
l: desc.get_values(ieml=ieml, language=l, descriptor=descriptor)
for l in LANGUAGES
}
if all(sorted(value[l]) == sorted(old_trans[l]) for l in LANGUAGES):
error("No update needed, db already contains {}:{} for {}".format(
descriptor, json.dumps(value), str(ieml)))
return False
# test if after modification there is still at least a descriptor
if all(not (desc.get_values(ieml=ieml, language=l, descriptor=d
) if d != descriptor else value[l])
for l in LANGUAGES for d in DESCRIPTORS_CLASS):
error('[descriptors] Remove {}'.format(str(ieml)))
with self.gitdb.commit(self.signature,
'[descriptors] Remove {}'.format(
str(ieml))):
db.remove_descriptor(ieml)
return True
# to_add = {l: [e for e in value[l] if e not in old_trans[l]] for l in LANGUAGES}
# to_remove = {l: [e for e in old_trans[l] if e not in value[l]] for l in LANGUAGES}
with self.gitdb.commit(
self.signature, '[descriptors] Update {} for {} to {}'.format(
descriptor, str(ieml), json.dumps(value))):
db.remove_descriptor(ieml, None, descriptor)
for l in LANGUAGES:
for e in value[l]:
db.add_descriptor(ieml, l, descriptor, e)
return True
def __exit__(self, type, value, traceback):
print("exiting commit context", file=stderr)
if type is None:
try:
status = self.db.status()
if not status:
return
index = self.db.repo.index
index.read()
for f, k in status.items():
if k & GIT_STATUS_WT_NEW:
index.add(f)
elif k & GIT_STATUS_WT_DELETED:
index.remove(f)
elif k & GIT_STATUS_WT_MODIFIED:
index.add(f)
elif k & GIT_STATUS_WT_RENAMED:
index.add(f)
index.write()
tree = index.write_tree()
oid = self.db.repo.create_commit(self.db.repo.head.name,
self.signature,
self.signature,
self.message,
tree,
[self.commit_id])
# self.db.commit_id = oid
error("committing db : {}".format(str(oid)))
except Exception as e:
error("Error commiting, reset to {}".format(self.commit_id))
self.db.reset(self.commit_id)
# TODO : ensure that the reset is perfect
# even when creating a new file in the folder ? untracked
raise e
else:
self.db.reset(self.commit_id)
def root(self, s):
try:
return self.table_to_root[self.tables[s]]
except KeyError as e:
error("No root defined for the script {}".format(str(s)))
return None
def pull(self, remote='origin') -> {str: Descriptor}:
"""
:param remote: the remote to pull from
:return: the conflicts : a dict that map ieml to the descriptor in conflict. In case of a conflict, the remote version is chosen as current version and the old current is returned by the function.
"""
repo = self.repo
remote_ = repo.remotes[remote]
try:
remote_.fetch(callbacks=pygit2.RemoteCallbacks(credentials=self.credentials))
except GitError as e:
logger.error(repr(e))
pass
current_commit = repo.head.target
# pull not the origin remote
# if self.target_commit is None or remote != 'origin':
# use most recent of remote
commit_target = repo.lookup_reference('refs/remotes/{}/{}'.format(remote, repo.head.shorthand)).target
# else:
# commit_target = self.target_commit
merge_result, _ = repo.merge_analysis(commit_target)
conflicts = {}
# Up to date, do nothing
if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
error("Merging: repository up-to-date")
# self.commit_id = commit_target
# We can just fastforward
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
error("Merging: fast-forward")
self.checkout(None, commit_target)
elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
error("Merging: cherry-pick local branch on remote branch")
# base = repo.merge_base(current_commit, commit_target)
# # rebased_commits : locals commits since base
# rebased_commits = []
# commit = repo.get(current_commit)
# while len(commit.parents):
# if base == commit.id:
# break
# rebased_commits.insert(0, commit)
# commit = commit.parents[0]
# branch = repo.branches.get(self.repo.head.shorthand)
#
# # checkout to pulled branch
# repo.checkout_tree(repo.get(commit_target),
# strategy=GIT_CHECKOUT_FORCE | GIT_CHECKOUT_RECREATE_MISSING)
# repo.head.set_target(commit_target)
repo.merge(commit_target)
# we are now in the remote state, we are going to try to add the local commit on top of the remote HEAD
# until a merge conflict
# last = commit_target
# for commit in tqdm.tqdm(rebased_commits, "Cherry picking commits"):
# repo.head.set_target(last)
# try:
# repo.cherrypick(commit.id)
# except GitError as e:
# # Cherry picking merge commit :
# # Usually you cannot cherry-pick a merge because you do not know which side of the merge should
# # be considered the mainline. This option specifies the parent number (starting from 1) of the
# # mainline and allows cherry-pick to replay the change relative to the specified parent.
# # https://stackoverflow.com/questions/9229301/git-cherry-pick-says-38c74d-is-a-merge-but-no-m-option-was-given
# pass
# # raise e
if repo.index.conflicts is None:
tree_id = repo.index.write_tree()
cherry = repo.get(commit_target)
last = repo.create_commit('HEAD', cherry.author, DEFAULT_COMMITER_SIGNATURE,
"Merge branch", tree_id, [repo.head.target, commit_target])
else:
to_commit = []
# ancestor : the IndexEntry on the file before the merge, or None if the file is created
# remote_entry : the IndexEntry of the merged remote file or None if remotely deleted
# local_entry : the IndexEntry of the local file or None if locally deleted
# resolve conflicts ...
for ancestor, local_entry, remote_entry in repo.index.conflicts:
old_path = ancestor.path if ancestor is not None else local_entry.path
# None => deleted
new_path = remote_entry.path if remote_entry is not None else None
if new_path is not None and old_path != new_path:
raise ValueError("Renaming not supported")
# add local entry as conflicts if a descriptor
if old_path.endswith('.desc'):
if local_entry is not None:
if ancestor is not None and ancestor.path != local_entry.path:
raise ValueError("Renaming not supported")
# local entry is not deleted
res = Descriptors.from_csv_string(repo.get(local_entry.oid).data.decode('utf8'),
assert_unique_ieml=True)
ieml = next(iter(res))
conflicts[ieml] = res[ieml]
else:
# locally deleted
assert remote_entry is not None
res = Descriptors.from_csv_string(repo.get(remote_entry.oid).data.decode('utf8'),
assert_unique_ieml=True)
ieml = next(iter(res))
conflicts[ieml] = {d: {l : [] for l in LANGUAGES} for d in DESCRIPTORS_CLASS}
else:
print("Ignoring", old_path, "not a descriptor", file=stderr)
# repo.index.read()
to_commit.append({
'old_path': old_path,
'new_path': new_path,
'data': repo.get(remote_entry.oid).data if remote_entry is not None else None
})
if to_commit != []:
for comm in to_commit:
data = comm['data']
old_path = comm['old_path']
new_path = comm['new_path']
# accept theirs (ours from git point of view)
if data is not None:
with open(os.path.join(self.folder, new_path), 'wb') as fp:
fp.write(data)
repo.index.add(new_path)
if new_path is None:
del repo.index.conflicts[old_path]
os.remove(os.path.join(self.folder, old_path))
repo.index.write()
tree_id = repo.index.write_tree()
last = self.repo.create_commit('HEAD',
DEFAULT_COMMITER_SIGNATURE,
DEFAULT_COMMITER_SIGNATURE,
"Merge {}".format(ieml),
tree_id,
[repo.head.target, commit_target])
# repo.state_cleanup()
# repo.head.set_target(last)
else:
#TODO handle merge conflicts here
raise ValueError("Incompatible history, can't merge origin into {}#{} in folder {}".format(self.branch, commit_target,self.folder))
return conflicts