def test_get_diff(self):
req_path = "project/+/deadbeef..master/"
self.mock_fetch(base64.b64encode("thepatch"))
patch = gitiles.get_diff(HOSTNAME, "project", "deadbeef", "master", "/")
self.assertEqual(patch, "thepatch")
gerrit.fetch_async.assert_called_once_with(HOSTNAME, req_path, headers={"Accept": "text/plain"})
def test_get_diff(self):
req_path = 'project/+/deadbeef..master/'
self.mock_fetch(base64.b64encode('thepatch'))
patch = gitiles.get_diff(HOSTNAME, 'project', 'deadbeef', 'master',
'/')
self.assertEqual(patch, 'thepatch')
gerrit.fetch_async.assert_called_once_with(
HOSTNAME, req_path, headers={'Accept': 'text/plain'})
def test_get_diff(self):
req_path = 'project/+/deadbeef..master/'
self.mock_fetch(base64.b64encode('thepatch'))
patch = gitiles.get_diff(HOSTNAME, 'project', 'deadbeef', 'master', '/')
self.assertEqual(patch, 'thepatch')
gerrit.fetch_async.assert_called_once_with(
HOSTNAME,
req_path,
headers={'Accept': 'text/plain'})
def _task_update(ref_id):
"""Updates documents within a ref. Idempotent."""
hostname, project, ref_name = parse_ref_id(ref_id)
loc = ref_id_to_loc(ref_id)
ref = Ref.get_by_id(ref_id)
from_commit = ref.indexed_revision if ref else None
to_commit = ref.indexing_revision if ref else None
logging.info('Updating %s: %r -> %r', ref_id, from_commit, to_commit)
if from_commit == to_commit:
if to_commit is not None:
logging.info('Already indexed')
return
# Index state is undefined and documents are requested to be deleted.
if from_commit and to_commit:
diff = gitiles.get_diff(hostname, project, from_commit, to_commit, '/')
if diff is None:
# May happen if |from_commit| does not exist anymore (git push --force)
logging.warning('Could not load diff %s..%s', from_commit, to_commit)
else:
deleted = set()
added = set()
deleted_prefix = '--- a/'
added_prefix = '+++ b/'
for line in diff.splitlines():
if line.startswith(deleted_prefix) and line.lower().endswith('.md'):
deleted.add(line[len(deleted_prefix):])
elif line.startswith(added_prefix) and line.lower().endswith('.md'):
added.add(line[len(added_prefix):])
logging.debug('%d deleted: %r', len(deleted), list(sorted(deleted)))
logging.debug('%d added: %r', len(added), list(sorted(added)))
INDEX.delete(['%s/%s' % (ref_id, path) for path in deleted])
added_locs = map(loc.join, added)
INDEX.put(_load_docs_async(added_locs, to_commit).get_result())
_mark_as_indexed(ref_id, from_commit, to_commit)
return
if to_commit:
logging.warning('Doing a full refresh')
_delete_all(hostname, project, ref_name)
# Crawling an entire tree and adding all documents may not be performed in a
# single run, so enqueue a task that will eventually add them all.
deferred.defer(_task_add_all, ref_id, to_commit)
else:
logging.warning('Deleting all docs')
_delete_all(hostname, project, ref_name)
_mark_as_indexed(ref_id, from_commit, to_commit)