def test_get_file_caching(self):
"""Testing Repository.get_file caches result"""
path = 'readme'
revision = 'abc123'
base_commit_id = 'def456'
repository = self.repository
scmtool_cls = repository.scmtool_class
self.spy_on(scmtool_cls.get_file,
owner=scmtool_cls,
op=kgb.SpyOpReturn(b'file data'))
# Two requests to the same path/revision should result in one only
# call.
data1 = repository.get_file(path=path, revision=revision)
data2 = repository.get_file(path=path,
revision=revision,
context=FileLookupContext())
self.assertIsInstance(data1, bytes)
self.assertIsInstance(data2, bytes)
self.assertEqual(data1, b'file data')
self.assertEqual(data1, data2)
self.assertSpyCallCount(scmtool_cls.get_file, 1)
self.assertSpyLastCalledWith(scmtool_cls.get_file,
path,
revision=revision,
base_commit_id=None)
# A base commit ID should result in a new call.
data3 = repository.get_file(path=path,
revision=revision,
base_commit_id=base_commit_id)
self.assertEqual(data3, data1)
self.assertSpyCallCount(scmtool_cls.get_file, 2)
self.assertSpyLastCalledWith(scmtool_cls.get_file,
path,
revision=revision,
base_commit_id=base_commit_id)
# Another fetch with the same base_commit_id will use the cached
# version, even if specified in a FileLookupContext.
context = FileLookupContext(base_commit_id=base_commit_id)
data4 = repository.get_file(path=path,
revision=revision,
context=context)
self.assertEqual(data4, data1)
self.assertSpyCallCount(scmtool_cls.get_file, 2)
def test_get_file_exists_caching_when_not_exists(self):
"""Testing Repository.get_file_exists doesn't cache result when the
file does not exist
"""
path = 'readme'
revision = '12345'
base_commit_id = 'def456'
repository = self.repository
scmtool_cls = repository.scmtool_class
self.spy_on(scmtool_cls.file_exists,
owner=scmtool_cls,
op=kgb.SpyOpReturn(False))
context = FileLookupContext(base_commit_id=base_commit_id)
self.assertFalse(
repository.get_file_exists(path=path, revision=revision))
self.assertFalse(
repository.get_file_exists(path=path,
revision=revision,
context=FileLookupContext()))
self.assertFalse(
repository.get_file_exists(path=path,
revision=revision,
base_commit_id=base_commit_id))
self.assertFalse(
repository.get_file_exists(path=path,
revision=revision,
context=context))
self.assertSpyCallCount(scmtool_cls.file_exists, 4)
self.assertSpyCalledWith(scmtool_cls.file_exists.calls[0],
path,
revision=revision,
base_commit_id=None)
self.assertSpyCalledWith(scmtool_cls.file_exists.calls[1],
path,
revision=revision,
base_commit_id=None)
self.assertSpyCalledWith(scmtool_cls.file_exists.calls[2],
path,
revision=revision,
base_commit_id=base_commit_id)
self.assertSpyCalledWith(scmtool_cls.file_exists.calls[3],
path,
revision=revision,
base_commit_id=base_commit_id,
context=context)
def test_get_file_exists_caching_when_exists(self):
"""Testing Repository.get_file_exists caches result when exists"""
path = 'readme'
revision = 'e965047'
base_commit_id = 'def456'
repository = self.repository
scmtool_cls = repository.scmtool_class
self.spy_on(scmtool_cls.file_exists,
owner=scmtool_cls,
op=kgb.SpyOpReturn(True))
# Two requests to the same path/revision should result in one only
# call.
self.assertTrue(
repository.get_file_exists(path=path, revision=revision))
self.assertTrue(
repository.get_file_exists(path=path,
revision=revision,
context=FileLookupContext()))
self.assertSpyCallCount(scmtool_cls.file_exists, 1)
self.assertSpyLastCalledWith(scmtool_cls.file_exists,
path,
revision=revision,
base_commit_id=None)
# A base commit ID should result in a new call.
self.assertTrue(
repository.get_file_exists(path=path,
revision=revision,
base_commit_id=base_commit_id))
self.assertSpyCallCount(scmtool_cls.file_exists, 2)
self.assertSpyLastCalledWith(scmtool_cls.file_exists,
path,
revision=revision,
base_commit_id=base_commit_id)
# Another check with the same base_commit_id will use the cached
# version, even if specified in a FileLookupContext.
context = FileLookupContext(base_commit_id=base_commit_id)
self.assertTrue(
repository.get_file_exists(path=path,
revision=revision,
context=context))
self.assertSpyCallCount(scmtool_cls.file_exists, 2)
def test_get_file_exists_caching_with_fetched_file(self):
"""Testing Repository.get_file_exists uses get_file's cached result"""
path = 'readme'
revision = 'abc123'
base_commit_id = 'def456'
repository = self.repository
scmtool_cls = repository.scmtool_class
self.spy_on(scmtool_cls.get_file,
owner=scmtool_cls,
op=kgb.SpyOpReturn(b'file data'))
self.spy_on(scmtool_cls.file_exists,
owner=scmtool_cls,
op=kgb.SpyOpReturn(True))
# These requests to the same path/revision should result in one only
# call.
repository.get_file(path=path, revision=revision)
self.assertTrue(
repository.get_file_exists(path=path, revision=revision))
self.assertTrue(
repository.get_file_exists(path=path,
revision=revision,
context=FileLookupContext()))
self.assertSpyCallCount(scmtool_cls.get_file, 1)
self.assertSpyNotCalled(scmtool_cls.file_exists)
# A base commit ID should result in a new call, which should then
# persist for file checks.
repository.get_file(path=path,
revision=revision,
base_commit_id=base_commit_id)
self.assertTrue(
repository.get_file_exists(path=path,
revision=revision,
base_commit_id=base_commit_id))
self.assertTrue(
repository.get_file_exists(
path=path,
revision=revision,
context=FileLookupContext(base_commit_id=base_commit_id)))
self.assertSpyCallCount(scmtool_cls.get_file, 2)
self.assertSpyNotCalled(scmtool_cls.file_exists)
def test_get_file_with_context(self):
"""Testing Repository.get_file with context="""
repository = self.repository
scmtool_cls = repository.scmtool_class
self.spy_on(scmtool_cls.get_file,
owner=scmtool_cls,
op=kgb.SpyOpReturn(b'data'))
context = FileLookupContext(base_commit_id='def456')
repository.get_file(path='readme', revision='abc123', context=context)
self.assertSpyCalledWith(scmtool_cls.get_file,
'readme',
revision='abc123',
base_commit_id='def456',
context=context)
def test_get_file_exists_signals(self):
"""Testing Repository.get_file_exists emits signals"""
def on_checking(**kwargs):
pass
def on_checked(**kwargs):
pass
repository = self.repository
checking_file_exists.connect(on_checking, sender=repository)
checked_file_exists.connect(on_checked, sender=repository)
self.spy_on(on_checking)
self.spy_on(on_checked)
path = 'readme'
revision = 'e965047'
base_commit_id = 'def456'
request = self.create_http_request()
context = FileLookupContext(request=request,
base_commit_id=base_commit_id)
repository.get_file_exists(path=path,
revision=revision,
context=context)
self.assertSpyCalledWith(on_checking,
sender=repository,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
context=context)
self.assertSpyCalledWith(on_checked,
sender=repository,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
context=context)
def test_get_file_signals(self):
"""Testing Repository.get_file emits signals"""
def on_fetching_file(**kwargs):
pass
def on_fetched_file(**kwargs):
pass
repository = self.repository
fetching_file.connect(on_fetching_file, sender=repository)
fetched_file.connect(on_fetched_file, sender=repository)
self.spy_on(on_fetching_file)
self.spy_on(on_fetched_file)
path = 'readme'
revision = 'e965047'
base_commit_id = 'def456'
request = self.create_http_request()
context = FileLookupContext(request=request,
base_commit_id=base_commit_id)
repository.get_file(path=path, revision=revision, context=context)
self.assertSpyCalledWith(on_fetching_file,
sender=repository,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
context=context)
self.assertSpyCalledWith(on_fetched_file,
sender=repository,
path=path,
revision=revision,
base_commit_id=base_commit_id,
request=request,
context=context,
data=b'Hello\n')
def _process_files(parsed_diff,
basedir,
repository,
base_commit_id,
request,
get_file_exists=None,
check_existence=False,
limit_to=None):
"""Collect metadata about files in the parser.
Args:
parsed_diff (reviewboard.diffviewer.parser.ParsedDiff):
The parsed diff to process.
basedir (unicode):
The base directory to prepend to all file paths in the diff.
repository (reviewboard.scmtools.models.Repository):
The repository that the diff was created against.
base_commit_id (unicode):
The ID of the commit that the diff is based upon. This is
needed by some SCMs or hosting services to properly look up
files, if the diffs represent blob IDs instead of commit IDs
and the service doesn't support those lookups.
request (django.http.HttpRequest):
The current HTTP request.
check_existence (bool, optional):
Whether or not existence checks should be performed against
the upstream repository.
get_file_exists (callable, optional):
A callable to use to determine if a given file exists in the
repository.
If ``check_existence`` is ``True`` this argument must be
provided.
limit_to (list of unicode, optional):
A list of filenames to limit the results to.
Yields:
reviewboard.diffviewer.parser.ParsedDiffFile:
Each file present in the diff.
Raises:
ValueError:
``check_existence`` was ``True`` but ``get_file_exists`` was not
provided.
"""
if check_existence and get_file_exists is None:
raise ValueError('Must provide get_file_exists when check_existence '
'is True')
tool = repository.get_scmtool()
basedir = force_bytes(basedir)
parsed_change = parsed_diff.changes[0]
for f in parsed_change.files:
# This will either be a Revision or bytes. Either way, convert it
# bytes now.
orig_revision = force_bytes(f.orig_file_details)
source_filename, source_revision = tool.parse_diff_revision(
f.orig_filename, orig_revision, moved=f.moved, copied=f.copied)
assert isinstance(source_filename, bytes), (
'%s.parse_diff_revision() must return a bytes filename, not %r' %
(type(tool).__name__, type(source_filename)))
assert isinstance(source_revision, (bytes, Revision)), (
'%s.parse_diff_revision() must return a revision which is either '
'bytes or reviewboard.scmtools.core.Revision, not %r' %
(type(tool).__name__, type(source_revision)))
dest_filename = _normalize_filename(f.modified_filename, basedir)
if limit_to is not None and dest_filename not in limit_to:
# This file isn't actually needed for the diff, so save
# ourselves a remote file existence check and some storage.
continue
source_filename = _normalize_filename(source_filename, basedir)
if (check_existence and source_revision not in (PRE_CREATION, UNKNOWN)
and not f.binary and not f.deleted and not f.moved
and not f.copied):
context = FileLookupContext(
request=request,
base_commit_id=base_commit_id,
diff_extra_data=parsed_diff.extra_data,
commit_extra_data=parsed_change.extra_data,
file_extra_data=f.extra_data)
if not get_file_exists(path=force_text(source_filename),
revision=force_text(source_revision),
context=context):
raise FileNotFoundError(path=force_text(source_filename),
revision=force_text(source_revision),
base_commit_id=base_commit_id,
context=context)
f.orig_filename = source_filename
f.orig_file_details = source_revision
f.modified_filename = dest_filename
yield f
def get_file_exists(self,
path,
revision,
base_commit_id=None,
request=None,
context=None):
"""Return whether or not a file exists in the repository.
If the repository is backed by a hosting service, this will go
through that. Otherwise, it will attempt to directly access the
repository.
The result of this call will be cached, making future lookups
of this path and revision on this repository faster.
This will send the
:py:data:`~reviewboard.scmtools.signals.checking_file_exists` signal
before beginning a file fetch from the repository (if not cached), and
the :py:data:`~reviewboard.scmtools.signals.checked_file_exists` signal
after.
Args:
path (unicode):
The path to the file in the repository.
revision (unicode);
The revision of the file to check.
base_commit_id (unicode, optional):
The ID of the commit containing the revision of the file
to check. This is required for some types of repositories
where the revision of a file and the ID of a commit differ.
Deprecated:
4.0.5:
Callers should provide this in ``context`` instead.
request (django.http.HttpRequest, optional):
The current HTTP request from the client. This is used for
logging purposes.
Deprecated:
4.0.5:
Callers should provide this in ``context`` instead.
context (reviewboard.scmtools.core.FileLookupContext, optional):
Extra context used to help look up this file.
This contains information about the HTTP request, requesting
user, and parsed diff information, which may be useful as
part of the repository lookup process.
Version Added:
4.0.5
Returns:
bool:
``True`` if the file exists in the repository. ``False`` if it
does not.
Raises:
TypeError:
One or more of the provided arguments is an invalid type.
Details are contained in the error message.
"""
if not isinstance(path, str):
raise TypeError('"path" must be a Unicode string, not %s' %
type(path))
if not isinstance(revision, str):
raise TypeError('"revision" must be a Unicode string, not %s' %
type(revision))
if context is None:
# If an explicit context isn't provided, create one. In a future
# version, this will be required.
context = FileLookupContext(request=request,
base_commit_id=base_commit_id)
key = self._make_file_exists_cache_key(
path=path,
revision=revision,
base_commit_id=context.base_commit_id)
if cache.get(make_cache_key(key)) == '1':
return True
exists = self._get_file_exists_uncached(path=path,
revision=revision,
context=context)
if exists:
cache_memoize(key, lambda: '1')
return exists
def get_file(self,
path,
revision,
base_commit_id=None,
request=None,
context=None):
"""Return a file from the repository.
This will attempt to retrieve the file from the repository. If the
repository is backed by a hosting service, it will go through that.
Otherwise, it will attempt to directly access the repository.
This will send the
:py:data:`~reviewboard.scmtools.signals.fetching_file` signal before
beginning a file fetch from the repository (if not cached), and the
:py:data:`~reviewboard.scmtools.signals.fetched_file` signal after.
Args:
path (unicode):
The path to the file in the repository.
revision (unicode):
The revision of the file to retrieve.
base_commit_id (unicode, optional):
The ID of the commit containing the revision of the file
to retrieve. This is required for some types of repositories
where the revision of a file and the ID of a commit differ.
Deprecated:
4.0.5:
Callers should provide this in ``context`` instead.
request (django.http.HttpRequest, optional):
The current HTTP request from the client. This is used for
logging purposes.
Deprecated:
4.0.5:
Callers should provide this in ``context`` instead.
context (reviewboard.scmtools.core.FileLookupContext, optional):
Extra context used to help look up this file.
This contains information about the HTTP request, requesting
user, and parsed diff information, which may be useful as
part of the repository lookup process.
Version Added:
4.0.5
Returns:
bytes:
The resulting file contents.
Raises:
TypeError:
One or more of the provided arguments is an invalid type.
Details are contained in the error message.
"""
# We wrap the result of get_file in a list and then return the first
# element after getting the result from the cache. This prevents the
# cache backend from converting to unicode, since we're no longer
# passing in a string and the cache backend doesn't recursively look
# through the list in order to convert the elements inside.
#
# Basically, this fixes the massive regressions introduced by the
# Django unicode changes.
if not isinstance(path, str):
raise TypeError('"path" must be a Unicode string, not %s' %
type(path))
if not isinstance(revision, str):
raise TypeError('"revision" must be a Unicode string, not %s' %
type(revision))
if context is None:
# If an explicit context isn't provided, create one. In a future
# version, this will be required.
context = FileLookupContext(request=request,
base_commit_id=base_commit_id)
return cache_memoize(
self._make_file_cache_key(path=path,
revision=revision,
base_commit_id=context.base_commit_id),
lambda: [
self._get_file_uncached(
path=path, revision=revision, context=context),
],
large_data=True)[0]