def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', 'source')
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, git_home)
# Enable calling fsync() for various operations touching .git
pygit2.option(pygit2.GIT_OPT_ENABLE_FSYNC_GITDIR, True)
addon_id = (addon_or_id.pk
if isinstance(addon_or_id, Addon) else addon_or_id)
self.git_repository_path = os.path.join(settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id),
package_type)
def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', 'source')
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.option(
pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL,
git_home)
# Enable calling fsync() for various operations touching .git
pygit2.option(pygit2.GIT_OPT_ENABLE_FSYNC_GITDIR, True)
addon_id = (
addon_or_id.pk
if isinstance(addon_or_id, Addon)
else addon_or_id)
self.git_repository_path = os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id),
package_type)
def test_search_path(self):
paths = [(GIT_CONFIG_LEVEL_GLOBAL, '/tmp/global'),
(GIT_CONFIG_LEVEL_XDG, '/tmp/xdg'),
(GIT_CONFIG_LEVEL_SYSTEM, '/tmp/etc')]
for level, path in paths:
option(GIT_OPT_SET_SEARCH_PATH, level, path)
self.assertEqual(path, option(GIT_OPT_GET_SEARCH_PATH, level))
def test_search_path():
paths = [(pygit2.GIT_CONFIG_LEVEL_GLOBAL, '/tmp/global'),
(pygit2.GIT_CONFIG_LEVEL_XDG, '/tmp/xdg'),
(pygit2.GIT_CONFIG_LEVEL_SYSTEM, '/tmp/etc')]
for level, path in paths:
option(pygit2.GIT_OPT_SET_SEARCH_PATH, level, path)
assert path == option(pygit2.GIT_OPT_GET_SEARCH_PATH, level)
def test_search_path(self):
paths = [(GIT_CONFIG_LEVEL_GLOBAL, '/tmp/global'),
(GIT_CONFIG_LEVEL_XDG, '/tmp/xdg'),
(GIT_CONFIG_LEVEL_SYSTEM, '/tmp/etc')]
for level, path in paths:
option(GIT_OPT_SET_SEARCH_PATH, level, path)
self.assertEqual(path, option(GIT_OPT_GET_SEARCH_PATH, level))
def empty_gitconfig(monkeypatch, tmpdir):
old = os.environ["HOME"]
(tmpdir / ".gitconfig").write_text("", encoding="utf8")
monkeypatch.setenv("HOME", str(tmpdir))
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, str(tmpdir))
yield
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, str(old))
def git_user_config(monkeypatch_session, tmp_path_factory, request):
home = tmp_path_factory.mktemp("home")
# override libgit2's search paths
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_SYSTEM, "")
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_XDG, "")
pygit2.option(
pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_GLOBAL, str(home)
)
# setup environment variables in case we call 'git' commands
monkeypatch_session.delenv("XDG_CONFIG_HOME", raising=False)
monkeypatch_session.setenv("HOME", str(home))
monkeypatch_session.setenv("GIT_ATTR_NOSYSTEM", "1")
monkeypatch_session.setenv("GIT_CONFIG_NOSYSTEM", "1")
USER_NAME = "Sno Tester"
USER_EMAIL = "*****@*****.**"
with open(home / ".gitconfig", "w") as f:
f.write(f"[user]\n\tname = {USER_NAME}\n\temail = {USER_EMAIL}\n")
L.debug("Temporary HOME for git config: %s", home)
with pytest.raises(IOError):
pygit2.Config.get_system_config()
global_cfg = pygit2.Config.get_global_config()
assert global_cfg["user.email"] == USER_EMAIL
assert global_cfg["user.name"] == USER_NAME
return (USER_EMAIL, USER_NAME, home)
def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', )
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, git_home)
# This will cause .keep file existence checks to be skipped when
# accessing packfiles, which can help performance with remote
# filesystems.
# See: https://github.com/mozilla/addons-server/issues/13019
pygit2.option(pygit2.GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, True)
# Enable calling fsync() for various operations touching .git
pygit2.option(pygit2.GIT_OPT_ENABLE_FSYNC_GITDIR, True)
self.addon_id = (addon_or_id.pk
if isinstance(addon_or_id, Addon) else addon_or_id)
self.git_repository_path = os.path.join(settings.GIT_FILE_STORAGE_PATH,
id_to_path(self.addon_id),
package_type)
def test_check_user_config(git_user_config, monkeypatch, data_archive, tmp_path):
# this is set by the global git_user_config fixture
u_email, u_name = check_git_user(repo=None)
assert u_email == git_user_config[0]
assert u_name == git_user_config[1]
# clear home
monkeypatch.setenv("HOME", str(tmp_path))
prev_home = pygit2.option(
pygit2.GIT_OPT_GET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_GLOBAL
)
try:
pygit2.option(
pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL,
str(tmp_path),
)
with data_archive("points"):
r = pygit2.Repository(".")
with pytest.raises(click.ClickException) as e:
check_git_user(repo=r)
assert "Please tell me who you are" in str(e)
subprocess.run(["git", "config", "user.name", "Alice"])
subprocess.run(["git", "config", "user.email", "*****@*****.**"])
check_git_user(repo=r)
with pytest.raises(click.ClickException) as e:
check_git_user(repo=None)
assert "Please tell me who you are" in str(e)
finally:
pygit2.option(
pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_GLOBAL, prev_home
)
def get_global_config_if_possible():
"""Try to return global git configuration, which normally lies in
`~/.gitconfig`.
However (https://github.com/libgit2/pygit2/issues/915),
`get_global_config()` fails, if the underlying file does not
exist yet. (The [paths may be
determined](https://github.com/libgit2/pygit2/issues/915#issuecomment-503300141)
by
`pygit2.option(pygit2.GIT_OPT_GET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_GLOBAL)`
and similar.)
Therefore, we do not simply `touch ~/.gitconfig` first, but
1. try `get_global_config()` (raises `IOError` in Python2, `OSError`
in Python3),
2. try `get_xdg_config()` (relying on the alternative global location
`$XDG_CONFIG_HOME/git/config`, typically aka `~/.config/git/config`
(this might fail due to the file not being there either (`OSError`,
`IOError`), or because the installed `libgit2`/`pygit2` is too old
(`AttributeError`; function added in 2014 only),
3. `touch ~/.gitconfig` and retry `get_global_config()`, and, as fallback
4. use the repo's `.git/config`, which should always be there."""
try:
return git.Config.get_global_config() # 1
except (IOError, OSError):
try:
return git.Config.get_xdg_config() # 2
except (IOError, OSError, AttributeError):
try:
sys.stderr.write("INFO: Creating global .gitconfig\n")
with open(os.path.join(
git.option(git.GIT_OPT_GET_SEARCH_PATH, git.GIT_CONFIG_LEVEL_GLOBAL),
'.gitconfig'), 'a'):
pass
return git.Config.get_global_config() # 3
except (IOError, OSError):
sys.stderr.write("INFO: Cannot record key ID in global config,"
" falling back to repo config\n")
return repo.config # 4
def git_user_config(monkeypatch_session, tmp_path_factory, request):
home = tmp_path_factory.mktemp("home")
# override libgit2's search paths
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_SYSTEM, "")
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_XDG,
"")
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, str(home))
# setup environment variables in case we call 'git' commands
monkeypatch_session.delenv("XDG_CONFIG_HOME", raising=False)
monkeypatch_session.setenv("HOME", str(home))
monkeypatch_session.setenv("GIT_ATTR_NOSYSTEM", "1")
monkeypatch_session.setenv("GIT_CONFIG_NOSYSTEM", "1")
USER_NAME = "Kart Tester"
USER_EMAIL = "*****@*****.**"
with open(home / ".gitconfig", "w") as f:
f.write(
f"[user]\n"
f"\tname = {USER_NAME}\n"
f"\temail = {USER_EMAIL}\n"
# make `gc` syncronous in testing.
# otherwise it has race conditions with test teardown.
f"[gc]\n"
f"\tautoDetach = false\n"
f"[init]\n"
f"\tdefaultBranch = main\n"
# used by test_clone_filter
f"[uploadPack]\n"
f"\tallowFilter = true\n")
L.debug("Temporary HOME for git config: %s", home)
with pytest.raises(IOError):
pygit2.Config.get_system_config()
global_cfg = pygit2.Config.get_global_config()
assert global_cfg["user.email"] == USER_EMAIL
assert global_cfg["user.name"] == USER_NAME
return (USER_EMAIL, USER_NAME, home)
def test_cached_memory():
value = option(pygit2.GIT_OPT_GET_CACHED_MEMORY)
assert value[1] == 256 * 1024**2
def test_cache_object_limit():
new_limit = 2 * 1024
option(pygit2.GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OBJ_BLOB, new_limit)
def __option(getter, setter, value):
old_value = option(getter)
option(setter, value)
assert value == option(getter)
# Reset to avoid side effects in later tests
option(setter, old_value)
# Write our various additions to $PATH
os.environ["PATH"] = (os.pathsep.join(path_extras) + os.pathsep +
os.environ.get("PATH", ""))
# GDAL Error Handling
from osgeo import gdal, ogr, osr
gdal.UseExceptions()
ogr.UseExceptions()
osr.UseExceptions()
# Libgit2 options
import pygit2
pygit2.option(pygit2.GIT_OPT_ENABLE_STRICT_HASH_VERIFICATION, 0)
# By default, libgit2 caches tree object reads (up to 4K trees).
# However, since Kart stores features in a 256x256 tree structure,
# large repos will have (65536+256) trees.
# Increasing this limit above that number increases import performance dramatically.
# (2 here is the value of `GIT_OBJECT_TREE` constant, pygit2 doesn't expose it)
pygit2.option(pygit2.GIT_OPT_SET_CACHE_OBJECT_LIMIT, 2, 100000)
# Libgit2 TLS CA Certificates
# We build libgit2 to prefer the OS certificate store on Windows/macOS, but Linux doesn't have one.
if is_linux:
import certifi
# note: cli_util.py also sets this in git's `http.sslCAInfo` config var
pygit2.settings.ssl_cert_file = certifi.where()
def test_write_tree_to(self):
pygit2.option(pygit2.GIT_OPT_ENABLE_STRICT_OBJECT_CREATION, False)
with utils.TemporaryRepository(('tar', 'emptyrepo')) as path:
nrepo = Repository(path)
id = self.repo.index.write_tree(nrepo)
assert nrepo[id] is not None
def test_enable_cache(self):
option(GIT_OPT_ENABLE_CACHING, False)
option(GIT_OPT_ENABLE_CACHING, True)
def test_mwindow_mapped_limit(self):
new_limit = 200 * 1024
option(GIT_OPT_SET_MWINDOW_MAPPED_LIMIT, new_limit)
self.assertEqual(new_limit, option(GIT_OPT_GET_MWINDOW_MAPPED_LIMIT))
def __option(self, getter, setter, value):
old_value = option(getter)
option(setter, value)
self.assertEqual(value, option(getter))
# Reset to avoid side effects in later tests
option(setter, old_value)
def test_mwindow_mapped_limit(self):
new_limit = 200 * 1024
option(GIT_OPT_SET_MWINDOW_MAPPED_LIMIT, new_limit)
self.assertEqual(new_limit, option(GIT_OPT_GET_MWINDOW_MAPPED_LIMIT))
def test_mwindow_size(self):
new_size = 200 * 1024
option(GIT_OPT_SET_MWINDOW_SIZE, new_size)
self.assertEqual(new_size, option(GIT_OPT_GET_MWINDOW_SIZE))
def __option(self, getter, setter, value):
old_value = option(getter)
option(setter, value)
assert value == option(getter)
# Reset to avoid side effects in later tests
option(setter, old_value)
def test_cached_memory(self):
value = option(GIT_OPT_GET_CACHED_MEMORY)
assert value[1] == 256 * 1024**2
def test_mwindow_size(self):
new_size = 200 * 1024
option(GIT_OPT_SET_MWINDOW_SIZE, new_size)
self.assertEqual(new_size, option(GIT_OPT_GET_MWINDOW_SIZE))
def test_enable_caching():
pygit2.settings.enable_caching(False)
pygit2.settings.enable_caching(True)
# Lower level API
option(pygit2.GIT_OPT_ENABLE_CACHING, False)
option(pygit2.GIT_OPT_ENABLE_CACHING, True)
def test_cached_memory(self):
value = option(GIT_OPT_GET_CACHED_MEMORY)
self.assertEqual(value[1], 256 * 1024**2)
def test_disable_pack_keep_file_checks():
pygit2.settings.disable_pack_keep_file_checks(False)
pygit2.settings.disable_pack_keep_file_checks(True)
# Lower level API
option(pygit2.GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, False)
option(pygit2.GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, True)
def test_enable_cache(self):
option(GIT_OPT_ENABLE_CACHING, False)
option(GIT_OPT_ENABLE_CACHING, True)
def test_cache_object_limit(self):
new_limit = 2 * 1024
option(GIT_OPT_SET_CACHE_OBJECT_LIMIT, GIT_OBJ_BLOB, new_limit)
def test_cached_memory(self):
value = option(GIT_OPT_GET_CACHED_MEMORY)
self.assertEqual(value[1], 256 * 1024**2)
