def get_diff_commits_origin_raw(self, ocean_backend):
"""Return the commit hashes which are stored in the raw index but not in the original repo.
:param ocean_backend: Ocean backend
"""
repo_origin = anonymize_url(self.perceval_backend.origin)
fltr = {
'name': 'origin',
'value': [repo_origin]
}
current_hashes = []
try:
git_repo = GitRepository(self.perceval_backend.uri, self.perceval_backend.gitpath)
current_hashes = [commit for commit in git_repo.rev_list()]
except EmptyRepositoryError:
logger.warning("No commits retrieved from {}, repo is empty".format(repo_origin))
except RepositoryError:
logger.warning("No commits retrieved from {}, repo doesn't exist locally".format(repo_origin))
except Exception as e:
logger.error("[git] No commits retrieved from {}, "
"git rev-list command failed: {}".format(repo_origin, e))
current_hashes = set(current_hashes)
raw_hashes = set([item['data']['commit']
for item in ocean_backend.fetch(ignore_incremental=True, _filter=fltr)])
hashes_to_delete = list(raw_hashes.difference(current_hashes))
return hashes_to_delete
def update_items(self, ocean_backend, enrich_backend):
"""Retrieve the commits not present in the original repository and delete
the corresponding documents from the raw and enriched indexes"""
fltr = {
'name': 'origin',
'value': [self.perceval_backend.origin]
}
logger.debug("[update-items] Checking commits for %s.", self.perceval_backend.origin)
git_repo = GitRepository(self.perceval_backend.uri, self.perceval_backend.gitpath)
try:
current_hashes = set([commit for commit in git_repo.rev_list()])
except Exception as e:
logger.error("Something went wrong with %s, %s", git_repo.uri, e, exc_info=True)
return
raw_hashes = set([item['data']['commit']
for item in ocean_backend.fetch(ignore_incremental=True, _filter=fltr)])
hashes_to_delete = list(raw_hashes.difference(current_hashes))
to_process = []
for _hash in hashes_to_delete:
to_process.append(_hash)
if len(to_process) != MAX_BULK_UPDATE_SIZE:
continue
# delete documents from the raw index
self.remove_commits(to_process, ocean_backend.elastic.index_url,
'data.commit', self.perceval_backend.origin)
# delete documents from the enriched index
self.remove_commits(to_process, enrich_backend.elastic.index_url,
'hash', self.perceval_backend.origin)
to_process = []
if to_process:
# delete documents from the raw index
self.remove_commits(to_process, ocean_backend.elastic.index_url,
'data.commit', self.perceval_backend.origin)
# delete documents from the enriched index
self.remove_commits(to_process, enrich_backend.elastic.index_url,
'hash', self.perceval_backend.origin)
logger.debug("[update-items] %s commits deleted from %s with origin %s.",
len(hashes_to_delete), ocean_backend.elastic.anonymize_url(ocean_backend.elastic.index_url),
self.perceval_backend.origin)
logger.debug("[update-items] %s commits deleted from %s with origin %s.",
len(hashes_to_delete), enrich_backend.elastic.anonymize_url(enrich_backend.elastic.index_url),
self.perceval_backend.origin)
# update branch info
self.delete_commit_branches(enrich_backend)
self.add_commit_branches(git_repo, enrich_backend)
def update_items(self, ocean_backend, enrich_backend):
"""Retrieve the commits not present in the original repository and delete
the corresponding documents from the raw and enriched indexes"""
fltr = {'name': 'origin', 'value': [self.perceval_backend.origin]}
logger.debug("[git] update-items Checking commits for {}.".format(
self.perceval_backend.origin))
try:
git_repo = GitRepository(self.perceval_backend.uri,
self.perceval_backend.gitpath)
current_hashes = set([commit for commit in git_repo.rev_list()])
except EmptyRepositoryError:
logger.warning("[git] Skip updating branch info for repo {}, "
"repo is empty".format(
self.perceval_backend.origin))
return
except RepositoryError:
logger.warning("[git] Skip updating branch info for repo {}, "
"repo doesn't exist locally".format(
self.perceval_backend.origin))
return
except Exception as e:
logger.error("[git] Skip updating branch info for repo {}, "
"git rev-list command failed: {}".format(
self.perceval_backend.origin, e))
return
raw_hashes = set([
item['data']['commit']
for item in ocean_backend.fetch(ignore_incremental=True,
_filter=fltr)
])
hashes_to_delete = list(raw_hashes.difference(current_hashes))
to_process = []
for _hash in hashes_to_delete:
to_process.append(_hash)
if len(to_process) != MAX_BULK_UPDATE_SIZE:
continue
# delete documents from the raw index
self.remove_commits(to_process, ocean_backend.elastic.index_url,
'data.commit', self.perceval_backend.origin)
# delete documents from the enriched index
self.remove_commits(to_process, enrich_backend.elastic.index_url,
'hash', self.perceval_backend.origin)
to_process = []
if to_process:
# delete documents from the raw index
self.remove_commits(to_process, ocean_backend.elastic.index_url,
'data.commit', self.perceval_backend.origin)
# delete documents from the enriched index
self.remove_commits(to_process, enrich_backend.elastic.index_url,
'hash', self.perceval_backend.origin)
logger.debug(
"[git] update-items {} commits deleted from {} with origin {}.".
format(
len(hashes_to_delete),
ocean_backend.elastic.anonymize_url(
ocean_backend.elastic.index_url),
self.perceval_backend.origin))
logger.debug(
"[git] update-items {} commits deleted from {} with origin {}.".
format(
len(hashes_to_delete),
enrich_backend.elastic.anonymize_url(
enrich_backend.elastic.index_url),
self.perceval_backend.origin))
