def _get_files(self, locale=None):
result = OrderedDict()
for path in get_all_files(self.dest):
path = force_text(path, errors='replace')
filename = os.path.basename(path)
short = path[len(self.dest) + 1:]
mime, encoding = mimetypes.guess_type(filename)
directory = os.path.isdir(path)
if not directory:
with open(path, 'rb') as fobj:
sha256 = get_sha256(fobj)
else:
sha256 = ''
result[short] = {
'id': self.file.id,
'binary': self._is_binary(mime, path),
'depth': short.count(os.sep),
'directory': directory,
'filename': filename,
'full': path,
'sha256': sha256,
'mimetype': mime or 'application/octet-stream',
'syntax': self.get_syntax(filename),
'modified': os.stat(path)[stat.ST_MTIME],
'short': short,
'size': os.stat(path)[stat.ST_SIZE],
'truncated': self.truncate(filename),
'version': self.file.version.version,
}
return result
def _get_files(self, locale=None):
result = OrderedDict()
for path in get_all_files(self.dest):
filename = force_text(os.path.basename(path), errors='replace')
short = force_text(path[len(self.dest) + 1:], errors='replace')
mime, encoding = mimetypes.guess_type(filename)
directory = os.path.isdir(path)
result[short] = {
'id': self.file.id,
'binary': self._is_binary(mime, path),
'depth': short.count(os.sep),
'directory': directory,
'filename': filename,
'full': path,
'sha256': get_sha256(path) if not directory else '',
'mimetype': mime or 'application/octet-stream',
'syntax': self.get_syntax(filename),
'modified': os.stat(path)[stat.ST_MTIME],
'short': short,
'size': os.stat(path)[stat.ST_SIZE],
'truncated': self.truncate(filename),
'version': self.file.version.version,
}
return result
def _check_dest_for_complete_listing(self, expected_files):
"""Check that all files we expect are in `self.dest`."""
dest_len = len(self.dest)
files_to_verify = get_all_files(self.dest)
difference = (
set([name[dest_len:].strip('/') for name in files_to_verify]) -
set(self._normalize_file_list(expected_files)))
return difference
def _check_dest_for_complete_listing(self, expected_files):
"""Check that all files we expect are in `self.dest`."""
dest_len = len(self.dest)
files_to_verify = get_all_files(self.dest)
difference = (
set([name[dest_len:].strip('/') for name in files_to_verify]) -
set(self._normalize_file_list(expected_files)))
return difference
def test_get_all_files_strip_prefix_no_prefix_silent():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
assert utils.get_all_files(tempdir, strip_prefix=tempdir) == [
os.path.join(tempdir, 'dir1'),
os.path.join(tempdir, 'dir1', 'foo2'),
os.path.join(tempdir, 'foo1'),
]
def test_get_all_files_prefix():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
assert utils.get_all_files(tempdir, prefix='/foo/bar') == [
'/foo/bar' + os.path.join(tempdir, 'dir1'),
'/foo/bar' + os.path.join(tempdir, 'dir1', 'foo2'),
'/foo/bar' + os.path.join(tempdir, 'foo1'),
]
def test_get_all_files_strip_prefix_no_prefix_silent():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
assert utils.get_all_files(tempdir, strip_prefix=tempdir) == [
os.path.join(tempdir, 'dir1'),
os.path.join(tempdir, 'dir1', 'foo2'),
os.path.join(tempdir, 'foo1'),
]
def test_get_all_files_prefix():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
assert utils.get_all_files(tempdir, prefix='/foo/bar') == [
'/foo/bar' + os.path.join(tempdir, 'dir1'),
'/foo/bar' + os.path.join(tempdir, 'dir1', 'foo2'),
'/foo/bar' + os.path.join(tempdir, 'foo1'),
]
def test_get_all_files_prefix_with_strip_prefix():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
result = utils.get_all_files(tempdir, strip_prefix=tempdir, prefix='/foo/bar')
assert result == [
os.path.join('/foo', 'bar', 'dir1'),
os.path.join('/foo', 'bar', 'dir1', 'foo2'),
os.path.join('/foo', 'bar', 'foo1'),
]
def test_get_all_files_prefix_with_strip_prefix():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
# strip_prefix alone doesn't do anything.
result = utils.get_all_files(
tempdir, strip_prefix=tempdir, prefix='/foo/bar')
assert result == [
os.path.join('/foo', 'bar', 'dir1'),
os.path.join('/foo', 'bar', 'dir1', 'foo2'),
os.path.join('/foo', 'bar', 'foo1'),
]
def test_get_all_files():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
assert utils.get_all_files(tempdir) == [
os.path.join(tempdir, 'dir1'),
os.path.join(tempdir, 'dir1', 'foo2'),
os.path.join(tempdir, 'foo1'),
]
shutil.rmtree(tempdir)
assert not os.path.exists(tempdir)
def test_get_all_files():
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
os.mkdir(os.path.join(tempdir, 'dir1'))
_touch(os.path.join(tempdir, 'foo1'))
_touch(os.path.join(tempdir, 'dir1', 'foo2'))
assert utils.get_all_files(tempdir) == [
os.path.join(tempdir, 'dir1'),
os.path.join(tempdir, 'dir1', 'foo2'),
os.path.join(tempdir, 'foo1'),
]
shutil.rmtree(tempdir)
assert not os.path.exists(tempdir)
def test_verify_files_calls_fsync_on_differences(self, fsync):
self.viewer.extract()
assert not fsync.called
files_to_verify = get_all_files(self.viewer.dest)
files_to_verify.pop()
with patch('olympia.files.helpers.get_all_files') as get_all_files_mck:
get_all_files_mck.return_value = files_to_verify
with pytest.raises(ValueError):
# We don't put things back into place after fsync
# so a `ValueError` is raised
self.viewer._verify_files(files_to_verify)
assert len(fsync.call_args_list) == len(files_to_verify) + 1
def test_verify_files_calls_fsync_on_differences(self, fsync):
self.viewer.extract()
assert not fsync.called
files_to_verify = get_all_files(self.viewer.dest)
files_to_verify.pop()
with patch('olympia.files.helpers.get_all_files') as get_all_files_mck:
get_all_files_mck.return_value = files_to_verify
with pytest.raises(ValueError):
# We don't put things back into place after fsync
# so a `ValueError` is raised
self.viewer._verify_files(files_to_verify)
assert len(fsync.call_args_list) == len(files_to_verify) + 1
def _get_files(self, locale=None):
"""We need the `locale` parameter for the memoization.
The `@memoize` decorator uses the prefix *and the parameters* to come
up with a memoize key. We thus add a (seemingly useless) `locale`
parameter.
Otherwise, we would just always have the urls for the files with the
locale from the first person checking them.
"""
result = OrderedDict()
for path in get_all_files(self.dest):
filename = force_text(os.path.basename(path), errors='replace')
short = force_text(path[len(self.dest) + 1:], errors='replace')
mime, encoding = mimetypes.guess_type(filename)
directory = os.path.isdir(path)
result[short] = {
'binary': self._is_binary(mime, path),
'depth': short.count(os.sep),
'directory': directory,
'filename': filename,
'full': path,
'sha256': get_sha256(path) if not directory else '',
'mimetype': mime or 'application/octet-stream',
'syntax': self.get_syntax(filename),
'modified': os.stat(path)[stat.ST_MTIME],
'short': short,
'size': os.stat(path)[stat.ST_SIZE],
'truncated': self.truncate(filename),
'url': reverse('files.list',
args=[self.file.id, 'file', short]),
'url_serve': reverse('files.redirect',
args=[self.file.id, short]),
'version': self.file.version.version,
}
return result
def _get_files(self, locale=None):
"""We need the `locale` parameter for the memoization.
The `@memoize` decorator uses the prefix *and the parameters* to come
up with a memoize key. We thus add a (seemingly useless) `locale`
parameter.
Otherwise, we would just always have the urls for the files with the
locale from the first person checking them.
"""
result = OrderedDict()
for path in get_all_files(self.dest):
filename = force_text(os.path.basename(path), errors='replace')
short = force_text(path[len(self.dest) + 1:], errors='replace')
mime, encoding = mimetypes.guess_type(filename)
directory = os.path.isdir(path)
result[short] = {
'binary': self._is_binary(mime, path),
'depth': short.count(os.sep),
'directory': directory,
'filename': filename,
'full': path,
'sha256': get_sha256(path) if not directory else '',
'mimetype': mime or 'application/octet-stream',
'syntax': self.get_syntax(filename),
'modified': os.stat(path)[stat.ST_MTIME],
'short': short,
'size': os.stat(path)[stat.ST_SIZE],
'truncated': self.truncate(filename),
'url': reverse('files.list',
args=[self.file.id, 'file', short]),
'url_serve': reverse('files.redirect',
args=[self.file.id, short]),
'version': self.file.version.version,
}
return result
def _commit_through_worktree(self, path, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
# Now extract the extension to the workdir
extract_extension_to_dest(
source=path,
dest=worktree.extraction_target_path,
force_fsync=True)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(
worktree.extraction_target_path,
worktree.path,
'')
# Make sure the index is up to date
worktree.repo.index.read()
for filename in files:
if os.path.basename(filename) == '.git':
# For security reasons git doesn't allow adding
# .git subdirectories anywhere in the repository.
# So we're going to rename them and add a random
# postfix
renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8])
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed)
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
return commit
def _commit_through_worktree(self, path, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
# Now extract the extension to the workdir
extract_extension_to_dest(
source=path,
dest=worktree.extraction_target_path,
force_fsync=True)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(
worktree.extraction_target_path,
worktree.path,
'')
# Make sure the index is up to date
worktree.repo.index.read()
for filename in files:
if os.path.basename(filename) == '.git':
# For security reasons git doesn't allow adding
# .git subdirectories anywhere in the repository.
# So we're going to rename them and add a random
# postfix
renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8])
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed)
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
return commit
def _commit_through_worktree(self, file_obj, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
if file_obj:
# Now extract the extension to the workdir
try:
extract_extension_to_dest(
source=file_obj.current_file_path,
dest=worktree.extraction_target_path,
force_fsync=True,
)
except FileNotFoundError:
extract_extension_to_dest(
source=file_obj.fallback_file_path,
dest=worktree.extraction_target_path,
force_fsync=True,
)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(worktree.extraction_target_path,
worktree.path, '')
# Make sure the index is up to date
worktree.repo.index.read()
# For security reasons git doesn't allow adding .git subdirectories
# anywhere in the repository. So we're going to rename them and add
# a random postfix.
# In order to disable the effect of the special git config files,
# we also have to postfix them.
files_to_rename = (
'.git',
'.gitattributes',
'.gitconfig',
'.gitignore',
'.gitmodules',
)
# Sort files by path length to rename the deepest files first.
files.sort(key=len, reverse=True)
for filename in files:
if os.path.basename(filename) in files_to_rename:
renamed = f'{filename}.{uuid.uuid4().hex[:8]}'
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed),
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user if possible.
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(),
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
#
# We use `lookup_reference` to fetch the most up-to-date
# reference to the branch in order to avoid an error described
# in: https://github.com/mozilla/addons-server/issues/13932
[self.git_repository.lookup_reference(branch.name).target],
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
#
# We use `lookup_reference` to fetch the most up-to-date reference
# to the branch in order to avoid an error described in:
# https://github.com/mozilla/addons-server/issues/13932
self.git_repository.lookup_reference(branch.name).set_target(
commit.hex)
return commit
