def test_full_run(self):
self._create_old_directory_structure()
for addon_id in (44, 160, 1789, 3000, 10202):
assert os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=1),
'addon',
))
assert not os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=2),
'addon',
))
stdout = io.StringIO()
call_command('migrate_git_storage_to_new_structure', stdout=stdout)
for addon_id in (44, 160, 1789, 3000, 10202):
# Old paths should no longer exist
assert not os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=1),
))
# New ones should.
assert os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=2),
))
stdout.seek(0)
assert stdout.read()
def __init__(self, addon_id, package_type='package'):
assert package_type in ('package', 'source')
self.git_repository_path = os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id),
package_type)
def test_extract_and_commit_from_version(settings):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
# Test that a new "unlisted" branch is created only if needed
addon.current_version.update(channel=amo.RELEASE_CHANNEL_UNLISTED)
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version)
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.git/config', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', 'source')
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = git_home
addon_id = (
addon_or_id.pk
if isinstance(addon_or_id, Addon)
else addon_or_id)
self.git_repository_path = os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id),
package_type)
def test_extract_and_commit_from_version(settings):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
# Test that a new "unlisted" branch is created only if needed
addon.current_version.update(channel=amo.RELEASE_CHANNEL_UNLISTED)
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version)
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def test_extract_and_commit_source_from_version(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from source file'.format(
repr(addon.current_version), addon.current_version.id, repr(addon))
assert expected in output
def test_extract_and_commit_from_version_valid_extensions(settings, filename):
addon = addon_factory(file_kw={'filename': filename})
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
# Make sure we are always calling fsync after extraction
assert fsync_mock.called
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', 'source')
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = git_home
addon_id = (
addon_or_id.pk
if isinstance(addon_or_id, Addon)
else addon_or_id)
self.git_repository_path = os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id),
package_type)
def __init__(self, addon_or_id, package_type='addon'):
from olympia.addons.models import Addon
assert package_type in ('addon', )
# Always enforce the search path being set to our ROOT
# setting. This is sad, libgit tries to fetch the global git
# config file (~/.gitconfig) and falls over permission errors while
# doing so in our web-environment.
# We are setting this here to avoid creating a unnecessary global
# state but since this is overwriting a global value in pygit2 it
# affects all pygit2 calls.
# https://github.com/libgit2/pygit2/issues/339
# https://github.com/libgit2/libgit2/issues/2122
git_home = settings.ROOT
pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH,
pygit2.GIT_CONFIG_LEVEL_GLOBAL, git_home)
# This will cause .keep file existence checks to be skipped when
# accessing packfiles, which can help performance with remote
# filesystems.
# See: https://github.com/mozilla/addons-server/issues/13019
pygit2.option(pygit2.GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, True)
# Enable calling fsync() for various operations touching .git
pygit2.option(pygit2.GIT_OPT_ENABLE_FSYNC_GITDIR, True)
self.addon_id = (addon_or_id.pk
if isinstance(addon_or_id, Addon) else addon_or_id)
self.git_repository_path = os.path.join(settings.GIT_FILE_STORAGE_PATH,
id_to_path(self.addon_id),
package_type)
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.git/config', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def test_extract_and_commit_source_from_version(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from source file'.format(
repr(addon.current_version), addon.current_version.id, repr(addon))
assert expected in output
def test_extract_and_commit_from_version_valid_extensions(settings, filename):
addon = addon_factory(file_kw={'filename': filename})
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
# Make sure we are always calling fsync after extraction
assert fsync_mock.called
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def test_basic(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
migrate_webextensions_to_git_storage([addon.pk])
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_basic(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
migrate_webextensions_to_git_storage([addon.pk])
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'package')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_full_run_dont_migrate(self):
self._create_old_directory_structure()
for addon_id in (44, 160, 1789, 3000, 10202):
assert os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=1),
))
assert not os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=2),
))
stdout = io.StringIO()
call_command('migrate_git_storage_to_new_structure',
'--dont-migrate',
stdout=stdout)
# Nothing should have been migrated...
for addon_id in (44, 160, 1789, 3000, 10202):
assert os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=1),
))
assert not os.path.exists(
os.path.join(
settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=2),
))
# New directory structure in temporary new storage path *should* have
# been created though, because it's needed to proceed.
for dir_pairs in (
('44', '0044'),
('60', '0160'),
('00', '3000'),
('02', '0202'),
):
assert os.path.exists(
os.path.join(settings.STORAGE_ROOT, 'new-git-storage',
*dir_pairs))
stdout.seek(0)
assert stdout.read()
def source_upload_path(instance, filename):
# At this point we already know that ext is one of VALID_SOURCE_EXTENSIONS
# because we already checked for that in
# /src/olympia/devhub/forms.py#WithSourceMixin.clean_source.
for ext in VALID_SOURCE_EXTENSIONS:
if filename.endswith(ext):
break
return os.path.join(
u'version_source', utils.id_to_path(instance.pk),
u'{0}-{1}-src{2}'.format(instance.addon.slug, instance.version, ext))
def test_extract_version_to_git(incr_mock, timer_mock):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
timer_mock.assert_any_call('git.extraction.version')
incr_mock.assert_called_with('git.extraction.version.success')
def test_extract_and_commit_from_file_obj_multiple_versions():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
repo = AddonGitRepository.extract_and_commit_from_file_obj(
addon.current_version.all_files[0], amo.RELEASE_CHANNEL_LISTED)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'package')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
env = {'GIT_DIR': repo.git_repository.path}
output = subprocess.check_output('git branch', shell=True, env=env)
assert 'listed' in output
output = subprocess.check_output('git log listed', shell=True, env=env)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
# Create two more versions, check that they appear in the comitlog
version = version_factory(addon=addon,
file_kw={'filename': 'webextension_no_id.xpi'},
version='0.2')
AddonGitRepository.extract_and_commit_from_file_obj(
file_obj=version.all_files[0], channel=amo.RELEASE_CHANNEL_LISTED)
version = version_factory(addon=addon,
file_kw={'filename': 'webextension_no_id.xpi'},
version='0.3')
repo = AddonGitRepository.extract_and_commit_from_file_obj(
file_obj=version.all_files[0], channel=amo.RELEASE_CHANNEL_LISTED)
output = subprocess.check_output('git log listed', shell=True, env=env)
assert output.count('Create new version') == 3
assert '0.1' in output
assert '0.2' in output
assert '0.3' in output
# 4 actual commits, including the repo initialization
assert output.count('Mozilla Add-ons Robot') == 4
# Make sure the commits didn't spill over into the master branch
output = subprocess.check_output('git log', shell=True, env=env)
assert output.count('Mozilla Add-ons Robot') == 1
assert '0.1' not in output
def get_sitemap_path(section, app, page=1):
if section is None or app is None:
# If we don't have a section or app, we don't need a complex directory
# structure and we can call the first page 'sitemap' for convenience
# (it's likely going to be the only page).
endpath = str(page) if page != 1 else 'sitemap'
else:
endpath = id_to_path(page)
return os.path.join(
settings.SITEMAP_STORAGE_PATH,
section or '',
app or '',
f'{endpath}.xml',
)
def source_upload_path(instance, filename):
# At this point we already know that ext is one of VALID_SOURCE_EXTENSIONS
# because we already checked for that in
# /src/olympia/devhub/forms.py#WithSourceMixin.clean_source.
for ext in VALID_SOURCE_EXTENSIONS:
if filename.endswith(ext):
break
return os.path.join(
u'version_source',
utils.id_to_path(instance.pk),
u'{0}-{1}-src{2}'.format(
instance.addon.slug,
instance.version,
ext)
)
def test_extract_and_commit_from_version_multiple_versions(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
# Create two more versions, check that they appear in the comitlog
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'},
version='0.2')
AddonGitRepository.extract_and_commit_from_version(version=version)
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'},
version='0.3')
repo = AddonGitRepository.extract_and_commit_from_version(version=version)
output = _run_process('git log listed', repo)
assert output.count('Create new version') == 3
assert '0.1' in output
assert '0.2' in output
assert '0.3' in output
# 4 actual commits, including the repo initialization
assert output.count('Mozilla Add-ons Robot') == 4
# Make sure the commits didn't spill over into the master branch
output = _run_process('git log', repo)
assert output.count('Mozilla Add-ons Robot') == 1
assert '0.1' not in output
def test_extract_version_to_git_deleted_version():
addon = addon_factory(file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
version = addon.current_version
version.delete()
hide_disabled_files()
extract_version_to_git(version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_source_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.update(source=source)
extract_version_source_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk, package_type='source')
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_source_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.update(source=source)
extract_version_source_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk, package_type='source')
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_id_to_path(value, expected):
assert utils.id_to_path(value) == expected
def test_id_to_path(value, expected):
assert utils.id_to_path(value) == expected
def get_new_path(self, addon_id):
return os.path.join(
self.get_new_temporary_file_storage_path(), id_to_path(addon_id, breadth=2)
)
def test_id_to_path_depth(value, expected):
assert utils.id_to_path(value, breadth=2) == expected
def _create_old_directory_structure(self):
# Pretend we have a bunch of add-ons with various ids.
for addon_id in (44, 160, 1789, 3000, 10202):
path = os.path.join(settings.GIT_FILE_STORAGE_PATH,
id_to_path(addon_id, breadth=1), 'addon')
os.makedirs(path)
