def test_extract_extension_to_dest_call_fsync(filename):
extension_file = f'src/olympia/files/fixtures/files/{filename}'
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
utils.extract_extension_to_dest(extension_file, force_fsync=True)
# fsync isn't called by default
assert fsync_mock.called
def test_extract_extension_to_dest_invalid_archive():
extension_file = 'src/olympia/files/fixtures/files/doesntexist.zip'
with mock.patch('olympia.files.utils.shutil.rmtree') as mock_rmtree:
with pytest.raises(forms.ValidationError):
utils.extract_extension_to_dest(extension_file)
# Make sure we are cleaning up our temprary directory if possible
assert mock_rmtree.called
def test_extract_extension_to_dest_call_fsync(filename):
extension_file = 'src/olympia/files/fixtures/files/{fname}'.format(
fname=filename)
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
utils.extract_extension_to_dest(extension_file, force_fsync=True)
# fsync isn't called by default
assert fsync_mock.called
def test_extract_extension_to_dest_invalid_archive():
extension_file = 'src/olympia/files/fixtures/files/doesntexist.zip'
with mock.patch('olympia.files.utils.shutil.rmtree') as mock_rmtree:
with pytest.raises(forms.ValidationError):
utils.extract_extension_to_dest(extension_file)
# Make sure we are cleaning up our temprary directory if possible
assert mock_rmtree.called
def _commit_through_worktree(self, path, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
# Now extract the extension to the workdir
extract_extension_to_dest(
source=path,
dest=worktree.path,
force_fsync=True)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
return commit
def test_extract_extension_to_dest(filename, expected_files):
extension_file = f'src/olympia/files/fixtures/files/{filename}'
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
temp_folder = utils.extract_extension_to_dest(extension_file)
assert sorted(os.listdir(temp_folder)) == expected_files
# fsync isn't called by default
assert not fsync_mock.called
def test_extract_extension_to_dest(filename, expected_files):
extension_file = 'src/olympia/files/fixtures/files/{fname}'.format(
fname=filename)
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
temp_folder = utils.extract_extension_to_dest(extension_file)
assert sorted(os.listdir(temp_folder)) == expected_files
# fsync isn't called by default
assert not fsync_mock.called
def _commit_through_worktree(self, file_obj, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
if file_obj:
# Now extract the extension to the workdir
try:
extract_extension_to_dest(
source=file_obj.current_file_path,
dest=worktree.extraction_target_path,
force_fsync=True,
)
except FileNotFoundError:
extract_extension_to_dest(
source=file_obj.fallback_file_path,
dest=worktree.extraction_target_path,
force_fsync=True,
)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(worktree.extraction_target_path,
worktree.path, '')
# Make sure the index is up to date
worktree.repo.index.read()
# For security reasons git doesn't allow adding .git subdirectories
# anywhere in the repository. So we're going to rename them and add
# a random postfix.
# In order to disable the effect of the special git config files,
# we also have to postfix them.
files_to_rename = (
'.git',
'.gitattributes',
'.gitconfig',
'.gitignore',
'.gitmodules',
)
# Sort files by path length to rename the deepest files first.
files.sort(key=len, reverse=True)
for filename in files:
if os.path.basename(filename) in files_to_rename:
renamed = f'{filename}.{uuid.uuid4().hex[:8]}'
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed),
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user if possible.
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(),
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
#
# We use `lookup_reference` to fetch the most up-to-date
# reference to the branch in order to avoid an error described
# in: https://github.com/mozilla/addons-server/issues/13932
[self.git_repository.lookup_reference(branch.name).target],
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
#
# We use `lookup_reference` to fetch the most up-to-date reference
# to the branch in order to avoid an error described in:
# https://github.com/mozilla/addons-server/issues/13932
self.git_repository.lookup_reference(branch.name).set_target(
commit.hex)
return commit
def _commit_through_worktree(self, path, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
# Now extract the extension to the workdir
extract_extension_to_dest(
source=path,
dest=worktree.extraction_target_path,
force_fsync=True)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(
worktree.extraction_target_path,
worktree.path,
'')
# Make sure the index is up to date
worktree.repo.index.read()
for filename in files:
if os.path.basename(filename) == '.git':
# For security reasons git doesn't allow adding
# .git subdirectories anywhere in the repository.
# So we're going to rename them and add a random
# postfix
renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8])
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed)
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
return commit
def _commit_through_worktree(self, path, message, author, branch):
"""
Create a temporary worktree that we can use to unpack the extension
without disturbing the current git workdir since it creates a new
temporary directory where we extract to.
"""
with TemporaryWorktree(self.git_repository) as worktree:
# Now extract the extension to the workdir
extract_extension_to_dest(
source=path,
dest=worktree.extraction_target_path,
force_fsync=True)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Fetch all files and strip the absolute path but keep the
# `extracted/` prefix
files = get_all_files(
worktree.extraction_target_path,
worktree.path,
'')
# Make sure the index is up to date
worktree.repo.index.read()
for filename in files:
if os.path.basename(filename) == '.git':
# For security reasons git doesn't allow adding
# .git subdirectories anywhere in the repository.
# So we're going to rename them and add a random
# postfix
renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8])
shutil.move(
os.path.join(worktree.path, filename),
os.path.join(worktree.path, renamed)
)
# Add all changes to the index (git add --all ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
self.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
self.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
return commit