def test_extract_extension_to_dest_call_fsync(filename): extension_file = f'src/olympia/files/fixtures/files/{filename}' with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: utils.extract_extension_to_dest(extension_file, force_fsync=True) # fsync isn't called by default assert fsync_mock.called
def test_extract_extension_to_dest_invalid_archive(): extension_file = 'src/olympia/files/fixtures/files/doesntexist.zip' with mock.patch('olympia.files.utils.shutil.rmtree') as mock_rmtree: with pytest.raises(forms.ValidationError): utils.extract_extension_to_dest(extension_file) # Make sure we are cleaning up our temprary directory if possible assert mock_rmtree.called
def test_extract_extension_to_dest_call_fsync(filename): extension_file = 'src/olympia/files/fixtures/files/{fname}'.format( fname=filename) with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: utils.extract_extension_to_dest(extension_file, force_fsync=True) # fsync isn't called by default assert fsync_mock.called
def test_extract_extension_to_dest_invalid_archive(): extension_file = 'src/olympia/files/fixtures/files/doesntexist.zip' with mock.patch('olympia.files.utils.shutil.rmtree') as mock_rmtree: with pytest.raises(forms.ValidationError): utils.extract_extension_to_dest(extension_file) # Make sure we are cleaning up our temprary directory if possible assert mock_rmtree.called
def _commit_through_worktree(self, path, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: # Now extract the extension to the workdir extract_extension_to_dest( source=path, dest=worktree.path, force_fsync=True) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target] ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) return commit
def test_extract_extension_to_dest(filename, expected_files): extension_file = f'src/olympia/files/fixtures/files/{filename}' with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: temp_folder = utils.extract_extension_to_dest(extension_file) assert sorted(os.listdir(temp_folder)) == expected_files # fsync isn't called by default assert not fsync_mock.called
def test_extract_extension_to_dest(filename, expected_files): extension_file = 'src/olympia/files/fixtures/files/{fname}'.format( fname=filename) with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: temp_folder = utils.extract_extension_to_dest(extension_file) assert sorted(os.listdir(temp_folder)) == expected_files # fsync isn't called by default assert not fsync_mock.called
def _commit_through_worktree(self, file_obj, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: if file_obj: # Now extract the extension to the workdir try: extract_extension_to_dest( source=file_obj.current_file_path, dest=worktree.extraction_target_path, force_fsync=True, ) except FileNotFoundError: extract_extension_to_dest( source=file_obj.fallback_file_path, dest=worktree.extraction_target_path, force_fsync=True, ) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Fetch all files and strip the absolute path but keep the # `extracted/` prefix files = get_all_files(worktree.extraction_target_path, worktree.path, '') # Make sure the index is up to date worktree.repo.index.read() # For security reasons git doesn't allow adding .git subdirectories # anywhere in the repository. So we're going to rename them and add # a random postfix. # In order to disable the effect of the special git config files, # we also have to postfix them. files_to_rename = ( '.git', '.gitattributes', '.gitconfig', '.gitignore', '.gitmodules', ) # Sort files by path length to rename the deepest files first. files.sort(key=len, reverse=True) for filename in files: if os.path.basename(filename) in files_to_rename: renamed = f'{filename}.{uuid.uuid4().hex[:8]}' shutil.move( os.path.join(worktree.path, filename), os.path.join(worktree.path, renamed), ) # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user if possible. self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log # # We use `lookup_reference` to fetch the most up-to-date # reference to the branch in order to avoid an error described # in: https://github.com/mozilla/addons-server/issues/13932 [self.git_repository.lookup_reference(branch.name).target], ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. # # We use `lookup_reference` to fetch the most up-to-date reference # to the branch in order to avoid an error described in: # https://github.com/mozilla/addons-server/issues/13932 self.git_repository.lookup_reference(branch.name).set_target( commit.hex) return commit
def _commit_through_worktree(self, path, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: # Now extract the extension to the workdir extract_extension_to_dest( source=path, dest=worktree.extraction_target_path, force_fsync=True) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Fetch all files and strip the absolute path but keep the # `extracted/` prefix files = get_all_files( worktree.extraction_target_path, worktree.path, '') # Make sure the index is up to date worktree.repo.index.read() for filename in files: if os.path.basename(filename) == '.git': # For security reasons git doesn't allow adding # .git subdirectories anywhere in the repository. # So we're going to rename them and add a random # postfix renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8]) shutil.move( os.path.join(worktree.path, filename), os.path.join(worktree.path, renamed) ) # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target] ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) return commit
def _commit_through_worktree(self, path, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: # Now extract the extension to the workdir extract_extension_to_dest( source=path, dest=worktree.extraction_target_path, force_fsync=True) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Fetch all files and strip the absolute path but keep the # `extracted/` prefix files = get_all_files( worktree.extraction_target_path, worktree.path, '') # Make sure the index is up to date worktree.repo.index.read() for filename in files: if os.path.basename(filename) == '.git': # For security reasons git doesn't allow adding # .git subdirectories anywhere in the repository. # So we're going to rename them and add a random # postfix renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8]) shutil.move( os.path.join(worktree.path, filename), os.path.join(worktree.path, renamed) ) # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target] ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) return commit