def test_full_run(self): self._create_old_directory_structure() for addon_id in (44, 160, 1789, 3000, 10202): assert os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=1), 'addon', )) assert not os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=2), 'addon', )) stdout = io.StringIO() call_command('migrate_git_storage_to_new_structure', stdout=stdout) for addon_id in (44, 160, 1789, 3000, 10202): # Old paths should no longer exist assert not os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=1), )) # New ones should. assert os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=2), )) stdout.seek(0) assert stdout.read()
def __init__(self, addon_id, package_type='package'): assert package_type in ('package', 'source') self.git_repository_path = os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id), package_type)
def test_extract_and_commit_from_version(settings): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) repo = AddonGitRepository.extract_and_commit_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' not in output # Test that a new "unlisted" branch is created only if needed addon.current_version.update(channel=amo.RELEASE_CHANNEL_UNLISTED) repo = AddonGitRepository.extract_and_commit_from_version( version=addon.current_version) output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') zip_file.writestr('.git/config', '') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock: uuid4_mock.return_value = mock.Mock( hex='b236f5994773477bbcd2d1b75ab1458f') repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git ls-tree -r --name-only listed', repo) assert set(output.split()) == { 'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def __init__(self, addon_or_id, package_type='addon'): from olympia.addons.models import Addon assert package_type in ('addon', 'source') # Always enforce the search path being set to our ROOT # setting. This is sad, libgit tries to fetch the global git # config file (~/.gitconfig) and falls over permission errors while # doing so in our web-environment. # We are setting this here to avoid creating a unnecessary global # state but since this is overwriting a global value in pygit2 it # affects all pygit2 calls. # https://github.com/libgit2/pygit2/issues/339 # https://github.com/libgit2/libgit2/issues/2122 git_home = settings.ROOT pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = git_home addon_id = ( addon_or_id.pk if isinstance(addon_or_id, Addon) else addon_or_id) self.git_repository_path = os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id), package_type)
def test_extract_and_commit_from_version(settings): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) repo = AddonGitRepository.extract_and_commit_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' not in output # Test that a new "unlisted" branch is created only if needed addon.current_version.update(channel=amo.RELEASE_CHANNEL_UNLISTED) repo = AddonGitRepository.extract_and_commit_from_version( version=addon.current_version) output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output
def test_extract_and_commit_source_from_version(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from source file'.format( repr(addon.current_version), addon.current_version.id, repr(addon)) assert expected in output
def test_extract_and_commit_from_version_valid_extensions(settings, filename): addon = addon_factory(file_kw={'filename': filename}) with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: repo = AddonGitRepository.extract_and_commit_from_version( addon.current_version) # Make sure we are always calling fsync after extraction assert fsync_mock.called assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' not in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output
def __init__(self, addon_or_id, package_type='addon'): from olympia.addons.models import Addon assert package_type in ('addon', 'source') # Always enforce the search path being set to our ROOT # setting. This is sad, libgit tries to fetch the global git # config file (~/.gitconfig) and falls over permission errors while # doing so in our web-environment. # We are setting this here to avoid creating a unnecessary global # state but since this is overwriting a global value in pygit2 it # affects all pygit2 calls. # https://github.com/libgit2/pygit2/issues/339 # https://github.com/libgit2/libgit2/issues/2122 git_home = settings.ROOT pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = git_home addon_id = ( addon_or_id.pk if isinstance(addon_or_id, Addon) else addon_or_id) self.git_repository_path = os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id), package_type)
def __init__(self, addon_or_id, package_type='addon'): from olympia.addons.models import Addon assert package_type in ('addon', ) # Always enforce the search path being set to our ROOT # setting. This is sad, libgit tries to fetch the global git # config file (~/.gitconfig) and falls over permission errors while # doing so in our web-environment. # We are setting this here to avoid creating a unnecessary global # state but since this is overwriting a global value in pygit2 it # affects all pygit2 calls. # https://github.com/libgit2/pygit2/issues/339 # https://github.com/libgit2/libgit2/issues/2122 git_home = settings.ROOT pygit2.option(pygit2.GIT_OPT_SET_SEARCH_PATH, pygit2.GIT_CONFIG_LEVEL_GLOBAL, git_home) # This will cause .keep file existence checks to be skipped when # accessing packfiles, which can help performance with remote # filesystems. # See: https://github.com/mozilla/addons-server/issues/13019 pygit2.option(pygit2.GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, True) # Enable calling fsync() for various operations touching .git pygit2.option(pygit2.GIT_OPT_ENABLE_FSYNC_GITDIR, True) self.addon_id = (addon_or_id.pk if isinstance(addon_or_id, Addon) else addon_or_id) self.git_repository_path = os.path.join(settings.GIT_FILE_STORAGE_PATH, id_to_path(self.addon_id), package_type)
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') zip_file.writestr('.git/config', '') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock: uuid4_mock.return_value = mock.Mock( hex='b236f5994773477bbcd2d1b75ab1458f') repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git ls-tree -r --name-only listed', repo) assert set(output.split()) == { 'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def test_extract_and_commit_source_from_version(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from source file'.format( repr(addon.current_version), addon.current_version.id, repr(addon)) assert expected in output
def test_extract_and_commit_from_version_valid_extensions(settings, filename): addon = addon_factory(file_kw={'filename': filename}) with mock.patch('olympia.files.utils.os.fsync') as fsync_mock: repo = AddonGitRepository.extract_and_commit_from_version( addon.current_version) # Make sure we are always calling fsync after extraction assert fsync_mock.called assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output assert 'unlisted' not in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output
def test_basic(self): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) migrate_webextensions_to_git_storage([addon.pk]) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_basic(self): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) migrate_webextensions_to_git_storage([addon.pk]) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'package') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_full_run_dont_migrate(self): self._create_old_directory_structure() for addon_id in (44, 160, 1789, 3000, 10202): assert os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=1), )) assert not os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=2), )) stdout = io.StringIO() call_command('migrate_git_storage_to_new_structure', '--dont-migrate', stdout=stdout) # Nothing should have been migrated... for addon_id in (44, 160, 1789, 3000, 10202): assert os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=1), )) assert not os.path.exists( os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=2), )) # New directory structure in temporary new storage path *should* have # been created though, because it's needed to proceed. for dir_pairs in ( ('44', '0044'), ('60', '0160'), ('00', '3000'), ('02', '0202'), ): assert os.path.exists( os.path.join(settings.STORAGE_ROOT, 'new-git-storage', *dir_pairs)) stdout.seek(0) assert stdout.read()
def source_upload_path(instance, filename): # At this point we already know that ext is one of VALID_SOURCE_EXTENSIONS # because we already checked for that in # /src/olympia/devhub/forms.py#WithSourceMixin.clean_source. for ext in VALID_SOURCE_EXTENSIONS: if filename.endswith(ext): break return os.path.join( u'version_source', utils.id_to_path(instance.pk), u'{0}-{1}-src{2}'.format(instance.addon.slug, instance.version, ext))
def test_extract_version_to_git(incr_mock, timer_mock): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] timer_mock.assert_any_call('git.extraction.version') incr_mock.assert_called_with('git.extraction.version.success')
def test_extract_and_commit_from_file_obj_multiple_versions(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) repo = AddonGitRepository.extract_and_commit_from_file_obj( addon.current_version.all_files[0], amo.RELEASE_CHANNEL_LISTED) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'package') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client env = {'GIT_DIR': repo.git_repository.path} output = subprocess.check_output('git branch', shell=True, env=env) assert 'listed' in output output = subprocess.check_output('git log listed', shell=True, env=env) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output # Create two more versions, check that they appear in the comitlog version = version_factory(addon=addon, file_kw={'filename': 'webextension_no_id.xpi'}, version='0.2') AddonGitRepository.extract_and_commit_from_file_obj( file_obj=version.all_files[0], channel=amo.RELEASE_CHANNEL_LISTED) version = version_factory(addon=addon, file_kw={'filename': 'webextension_no_id.xpi'}, version='0.3') repo = AddonGitRepository.extract_and_commit_from_file_obj( file_obj=version.all_files[0], channel=amo.RELEASE_CHANNEL_LISTED) output = subprocess.check_output('git log listed', shell=True, env=env) assert output.count('Create new version') == 3 assert '0.1' in output assert '0.2' in output assert '0.3' in output # 4 actual commits, including the repo initialization assert output.count('Mozilla Add-ons Robot') == 4 # Make sure the commits didn't spill over into the master branch output = subprocess.check_output('git log', shell=True, env=env) assert output.count('Mozilla Add-ons Robot') == 1 assert '0.1' not in output
def get_sitemap_path(section, app, page=1): if section is None or app is None: # If we don't have a section or app, we don't need a complex directory # structure and we can call the first page 'sitemap' for convenience # (it's likely going to be the only page). endpath = str(page) if page != 1 else 'sitemap' else: endpath = id_to_path(page) return os.path.join( settings.SITEMAP_STORAGE_PATH, section or '', app or '', f'{endpath}.xml', )
def source_upload_path(instance, filename): # At this point we already know that ext is one of VALID_SOURCE_EXTENSIONS # because we already checked for that in # /src/olympia/devhub/forms.py#WithSourceMixin.clean_source. for ext in VALID_SOURCE_EXTENSIONS: if filename.endswith(ext): break return os.path.join( u'version_source', utils.id_to_path(instance.pk), u'{0}-{1}-src{2}'.format( instance.addon.slug, instance.version, ext) )
def test_extract_and_commit_from_version_multiple_versions(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) repo = AddonGitRepository.extract_and_commit_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from {}'.format( repr(addon.current_version), addon.current_version.id, repr(addon), repr(addon.current_version.all_files[0])) assert expected in output # Create two more versions, check that they appear in the comitlog version = version_factory( addon=addon, file_kw={'filename': 'webextension_no_id.xpi'}, version='0.2') AddonGitRepository.extract_and_commit_from_version(version=version) version = version_factory( addon=addon, file_kw={'filename': 'webextension_no_id.xpi'}, version='0.3') repo = AddonGitRepository.extract_and_commit_from_version(version=version) output = _run_process('git log listed', repo) assert output.count('Create new version') == 3 assert '0.1' in output assert '0.2' in output assert '0.3' in output # 4 actual commits, including the repo initialization assert output.count('Mozilla Add-ons Robot') == 4 # Make sure the commits didn't spill over into the master branch output = _run_process('git log', repo) assert output.count('Mozilla Add-ons Robot') == 1 assert '0.1' not in output
def test_extract_version_to_git_deleted_version(): addon = addon_factory(file_kw={ 'filename': 'webextension_no_id.xpi', 'is_webextension': True }) version = addon.current_version version.delete() hide_disabled_files() extract_version_to_git(version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_source_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.update(source=source) extract_version_source_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk, package_type='source') assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_source_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.update(source=source) extract_version_source_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk, package_type='source') assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git']
def test_id_to_path(value, expected): assert utils.id_to_path(value) == expected
def test_id_to_path(value, expected): assert utils.id_to_path(value) == expected
def get_new_path(self, addon_id): return os.path.join( self.get_new_temporary_file_storage_path(), id_to_path(addon_id, breadth=2) )
def test_id_to_path_depth(value, expected): assert utils.id_to_path(value, breadth=2) == expected
def _create_old_directory_structure(self): # Pretend we have a bunch of add-ons with various ids. for addon_id in (44, 160, 1789, 3000, 10202): path = os.path.join(settings.GIT_FILE_STORAGE_PATH, id_to_path(addon_id, breadth=1), 'addon') os.makedirs(path)