def _get_files(self, locale=None): result = OrderedDict() for path in get_all_files(self.dest): path = force_text(path, errors='replace') filename = os.path.basename(path) short = path[len(self.dest) + 1:] mime, encoding = mimetypes.guess_type(filename) directory = os.path.isdir(path) if not directory: with open(path, 'rb') as fobj: sha256 = get_sha256(fobj) else: sha256 = '' result[short] = { 'id': self.file.id, 'binary': self._is_binary(mime, path), 'depth': short.count(os.sep), 'directory': directory, 'filename': filename, 'full': path, 'sha256': sha256, 'mimetype': mime or 'application/octet-stream', 'syntax': self.get_syntax(filename), 'modified': os.stat(path)[stat.ST_MTIME], 'short': short, 'size': os.stat(path)[stat.ST_SIZE], 'truncated': self.truncate(filename), 'version': self.file.version.version, } return result
def _get_files(self, locale=None): result = OrderedDict() for path in get_all_files(self.dest): filename = force_text(os.path.basename(path), errors='replace') short = force_text(path[len(self.dest) + 1:], errors='replace') mime, encoding = mimetypes.guess_type(filename) directory = os.path.isdir(path) result[short] = { 'id': self.file.id, 'binary': self._is_binary(mime, path), 'depth': short.count(os.sep), 'directory': directory, 'filename': filename, 'full': path, 'sha256': get_sha256(path) if not directory else '', 'mimetype': mime or 'application/octet-stream', 'syntax': self.get_syntax(filename), 'modified': os.stat(path)[stat.ST_MTIME], 'short': short, 'size': os.stat(path)[stat.ST_SIZE], 'truncated': self.truncate(filename), 'version': self.file.version.version, } return result
def _check_dest_for_complete_listing(self, expected_files): """Check that all files we expect are in `self.dest`.""" dest_len = len(self.dest) files_to_verify = get_all_files(self.dest) difference = ( set([name[dest_len:].strip('/') for name in files_to_verify]) - set(self._normalize_file_list(expected_files))) return difference
def test_get_all_files_strip_prefix_no_prefix_silent(): tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH) os.mkdir(os.path.join(tempdir, 'dir1')) _touch(os.path.join(tempdir, 'foo1')) _touch(os.path.join(tempdir, 'dir1', 'foo2')) # strip_prefix alone doesn't do anything. assert utils.get_all_files(tempdir, strip_prefix=tempdir) == [ os.path.join(tempdir, 'dir1'), os.path.join(tempdir, 'dir1', 'foo2'), os.path.join(tempdir, 'foo1'), ]
def test_get_all_files_prefix(): tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH) os.mkdir(os.path.join(tempdir, 'dir1')) _touch(os.path.join(tempdir, 'foo1')) _touch(os.path.join(tempdir, 'dir1', 'foo2')) # strip_prefix alone doesn't do anything. assert utils.get_all_files(tempdir, prefix='/foo/bar') == [ '/foo/bar' + os.path.join(tempdir, 'dir1'), '/foo/bar' + os.path.join(tempdir, 'dir1', 'foo2'), '/foo/bar' + os.path.join(tempdir, 'foo1'), ]
def test_get_all_files_prefix_with_strip_prefix(): tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH) os.mkdir(os.path.join(tempdir, 'dir1')) _touch(os.path.join(tempdir, 'foo1')) _touch(os.path.join(tempdir, 'dir1', 'foo2')) # strip_prefix alone doesn't do anything. result = utils.get_all_files(tempdir, strip_prefix=tempdir, prefix='/foo/bar') assert result == [ os.path.join('/foo', 'bar', 'dir1'), os.path.join('/foo', 'bar', 'dir1', 'foo2'), os.path.join('/foo', 'bar', 'foo1'), ]
def test_get_all_files_prefix_with_strip_prefix(): tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH) os.mkdir(os.path.join(tempdir, 'dir1')) _touch(os.path.join(tempdir, 'foo1')) _touch(os.path.join(tempdir, 'dir1', 'foo2')) # strip_prefix alone doesn't do anything. result = utils.get_all_files( tempdir, strip_prefix=tempdir, prefix='/foo/bar') assert result == [ os.path.join('/foo', 'bar', 'dir1'), os.path.join('/foo', 'bar', 'dir1', 'foo2'), os.path.join('/foo', 'bar', 'foo1'), ]
def test_get_all_files(): tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH) os.mkdir(os.path.join(tempdir, 'dir1')) _touch(os.path.join(tempdir, 'foo1')) _touch(os.path.join(tempdir, 'dir1', 'foo2')) assert utils.get_all_files(tempdir) == [ os.path.join(tempdir, 'dir1'), os.path.join(tempdir, 'dir1', 'foo2'), os.path.join(tempdir, 'foo1'), ] shutil.rmtree(tempdir) assert not os.path.exists(tempdir)
def test_verify_files_calls_fsync_on_differences(self, fsync): self.viewer.extract() assert not fsync.called files_to_verify = get_all_files(self.viewer.dest) files_to_verify.pop() with patch('olympia.files.helpers.get_all_files') as get_all_files_mck: get_all_files_mck.return_value = files_to_verify with pytest.raises(ValueError): # We don't put things back into place after fsync # so a `ValueError` is raised self.viewer._verify_files(files_to_verify) assert len(fsync.call_args_list) == len(files_to_verify) + 1
def _get_files(self, locale=None): """We need the `locale` parameter for the memoization. The `@memoize` decorator uses the prefix *and the parameters* to come up with a memoize key. We thus add a (seemingly useless) `locale` parameter. Otherwise, we would just always have the urls for the files with the locale from the first person checking them. """ result = OrderedDict() for path in get_all_files(self.dest): filename = force_text(os.path.basename(path), errors='replace') short = force_text(path[len(self.dest) + 1:], errors='replace') mime, encoding = mimetypes.guess_type(filename) directory = os.path.isdir(path) result[short] = { 'binary': self._is_binary(mime, path), 'depth': short.count(os.sep), 'directory': directory, 'filename': filename, 'full': path, 'sha256': get_sha256(path) if not directory else '', 'mimetype': mime or 'application/octet-stream', 'syntax': self.get_syntax(filename), 'modified': os.stat(path)[stat.ST_MTIME], 'short': short, 'size': os.stat(path)[stat.ST_SIZE], 'truncated': self.truncate(filename), 'url': reverse('files.list', args=[self.file.id, 'file', short]), 'url_serve': reverse('files.redirect', args=[self.file.id, short]), 'version': self.file.version.version, } return result
def _commit_through_worktree(self, path, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: # Now extract the extension to the workdir extract_extension_to_dest( source=path, dest=worktree.extraction_target_path, force_fsync=True) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Fetch all files and strip the absolute path but keep the # `extracted/` prefix files = get_all_files( worktree.extraction_target_path, worktree.path, '') # Make sure the index is up to date worktree.repo.index.read() for filename in files: if os.path.basename(filename) == '.git': # For security reasons git doesn't allow adding # .git subdirectories anywhere in the repository. # So we're going to rename them and add a random # postfix renamed = '{}.{}'.format(filename, uuid.uuid4().hex[:8]) shutil.move( os.path.join(worktree.path, filename), os.path.join(worktree.path, renamed) ) # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target] ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) return commit
def _commit_through_worktree(self, file_obj, message, author, branch): """ Create a temporary worktree that we can use to unpack the extension without disturbing the current git workdir since it creates a new temporary directory where we extract to. """ with TemporaryWorktree(self.git_repository) as worktree: if file_obj: # Now extract the extension to the workdir try: extract_extension_to_dest( source=file_obj.current_file_path, dest=worktree.extraction_target_path, force_fsync=True, ) except FileNotFoundError: extract_extension_to_dest( source=file_obj.fallback_file_path, dest=worktree.extraction_target_path, force_fsync=True, ) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Fetch all files and strip the absolute path but keep the # `extracted/` prefix files = get_all_files(worktree.extraction_target_path, worktree.path, '') # Make sure the index is up to date worktree.repo.index.read() # For security reasons git doesn't allow adding .git subdirectories # anywhere in the repository. So we're going to rename them and add # a random postfix. # In order to disable the effect of the special git config files, # we also have to postfix them. files_to_rename = ( '.git', '.gitattributes', '.gitconfig', '.gitignore', '.gitmodules', ) # Sort files by path length to rename the deepest files first. files.sort(key=len, reverse=True) for filename in files: if os.path.basename(filename) in files_to_rename: renamed = f'{filename}.{uuid.uuid4().hex[:8]}' shutil.move( os.path.join(worktree.path, filename), os.path.join(worktree.path, renamed), ) # Add all changes to the index (git add --all ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch oid = worktree.repo.create_commit( None, # author, using the actual uploading user if possible. self.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. self.get_author(), message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log # # We use `lookup_reference` to fetch the most up-to-date # reference to the branch in order to avoid an error described # in: https://github.com/mozilla/addons-server/issues/13932 [self.git_repository.lookup_reference(branch.name).target], ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. # # We use `lookup_reference` to fetch the most up-to-date reference # to the branch in order to avoid an error described in: # https://github.com/mozilla/addons-server/issues/13932 self.git_repository.lookup_reference(branch.name).set_target( commit.hex) return commit