def migrate_webextensions_to_git_storage(ids, **kw):
# recursive imports...
from olympia.versions.tasks import extract_version_to_git
log.info(
'Migrating add-ons to git storage %d-%d [%d].',
ids[0], ids[-1], len(ids))
addons = Addon.unfiltered.filter(id__in=ids)
for addon in addons:
# Filter out versions that are already present in the git
# storage.
versions = addon.versions.filter(git_hash='').order_by('created')
for version in versions:
# Back in the days an add-on was able to have multiple files
# per version. That changed, we are very naive here and extracting
# simply the first file in the list. For WebExtensions there is
# only a very very small number that have different files for
# a single version.
unique_file_hashes = set([
x.original_hash for x in version.all_files
])
if len(unique_file_hashes) > 1:
# Log actually different hashes so that we can clean them
# up manually and work together with developers later.
log.info(
'Version {version} of {addon} has more than one uploaded '
'file'.format(version=repr(version), addon=repr(addon)))
if not unique_file_hashes:
log.info('No files found for {version} from {addon}'.format(
version=repr(version), addon=repr(addon)))
continue
# Don't call the task as a task but do the extraction in process
# this makes sure we don't overwhelm the storage and also makes
# sure we don't end up with tasks committing at random times but
# correctly in-order instead.
try:
file_id = version.all_files[0].pk
log.info('Extracting file {file_id} to git storage'.format(
file_id=file_id))
extract_version_to_git(version.pk)
log.info(
'Extraction of file {file_id} into git storage succeeded'
.format(file_id=file_id))
except Exception:
log.exception(
'Extraction of file {file_id} from {version} '
'({addon}) failed'.format(
file_id=version.all_files[0],
version=repr(version),
addon=repr(addon)))
continue
def test_extract_version_to_git_with_not_extension_type():
addon = addon_factory(type=amo.ADDON_STATICTHEME)
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert not repo.is_extracted
def test_extract_version_to_git_with_non_webextension():
addon = addon_factory(type=amo.ADDON_EXTENSION,
file_kw={'is_webextension': False})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert not repo.is_extracted
def setUp(self):
super(TestFileEntriesSerializer, self).setUp()
self.addon = addon_factory(
file_kw={
'filename': 'notify-link-clicks-i18n.xpi',
'is_webextension': True})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
def setUp(self):
super(TestAddonCompareVersionSerializerFileOnly, self).setUp()
self.addon = addon_factory(
file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.reload()
self.version = self.addon.current_version
def setUp(self):
super(TestFileEntriesSerializer, self).setUp()
self.addon = addon_factory(
file_kw={
'filename': 'notify-link-clicks-i18n.xpi',
'is_webextension': True})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
def setUp(self):
super(TestFileEntriesDiffSerializer, self).setUp()
self.addon = addon_factory(
name=u'My Addôn', slug='my-addon',
file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
def test_extract_version_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git_with_cron_enabled_and_force_extraction():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository(addon.pk)
create_switch('enable-git-extraction-cron')
assert GitExtractionEntry.objects.count() == 0
extract_version_to_git(addon.current_version.pk, force_extraction=True)
assert GitExtractionEntry.objects.count() == 0
assert repo.is_extracted
def test_extract_version_to_git(incr_mock, timer_mock):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
timer_mock.assert_any_call('git.extraction.version')
incr_mock.assert_called_with('git.extraction.version.success')
def setUp(self):
super(TestAddonCompareVersionSerializer, self).setUp()
self.addon = addon_factory(name=u'My Addôn',
slug='my-addon',
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
self.version = self.addon.current_version
def setUp(self):
super(TestFileInfoSerializer, self).setUp()
self.addon = addon_factory(file_kw={
'filename': 'notify-link-clicks-i18n.xpi',
'is_webextension': True
})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.reload()
assert (self.addon.current_version.current_file.filename ==
'notify-link-clicks-i18n.xpi')
self.version = self.addon.current_version
self.file = self.addon.current_version.current_file
def test_extract_version_to_git_deleted_version():
addon = addon_factory(file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
version = addon.current_version
version.delete()
hide_disabled_files()
extract_version_to_git(version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_supports_search_plugin(self):
self.addon = addon_factory(file_kw={'filename': 'search_20190331.xml'})
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.refresh_from_db()
file = self.addon.current_version.current_file
data = self.serialize(file)
assert data['id'] == file.pk
assert set(data['entries'].keys()) == {'search_20190331.xml'}
assert data['selected_file'] == 'search_20190331.xml'
assert data['content'].startswith(
'<?xml version="1.0" encoding="utf-8"?>')
assert data['download_url'] == absolutify(
reverse('reviewers.download_git_file',
kwargs={
'version_id': self.addon.current_version.pk,
'filename': 'search_20190331.xml'
}))
def create_new_version_for_addon(self, xpi_filename):
addon = addon_factory(name=u'My Addôn',
slug='my-addon',
file_kw={'filename': xpi_filename})
extract_version_to_git(addon.current_version.pk)
addon.current_version.refresh_from_db()
parent_version = addon.current_version
new_version = version_factory(addon=addon,
file_kw={
'filename': xpi_filename,
'is_webextension': True,
})
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
return addon, repo, parent_version, new_version
def setUp(self):
super(TestAddonBrowseVersionSerializer, self).setUp()
license = License.objects.create(
name={
'en-US': u'My License',
'fr': u'Mä Licence',
},
text={
'en-US': u'Lorem ipsum dolor sit amet, has nemore patrioqué',
},
url='http://license.example.com/'
)
self.addon = addon_factory(
file_kw={
'hash': 'fakehash',
'is_mozilla_signed_extension': True,
'platform': amo.PLATFORM_ALL.id,
'size': 42,
'filename': 'notify-link-clicks-i18n.xpi',
'is_webextension': True
},
version_kw={
'license': license,
'min_app_version': '50.0',
'max_app_version': '*',
'release_notes': {
'en-US': u'Release notes in english',
'fr': u'Notes de version en français',
},
'reviewed': self.days_ago(0),
}
)
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.reload()
assert self.addon.current_version.release_notes
self.version = self.addon.current_version
def setUp(self):
super(TestAddonBrowseVersionSerializer, self).setUp()
license = License.objects.create(
name={
'en-US': u'My License',
'fr': u'Mä Licence',
},
text={
'en-US': u'Lorem ipsum dolor sit amet, has nemore patrioqué',
},
url='http://license.example.com/'
)
self.addon = addon_factory(
file_kw={
'hash': 'fakehash',
'is_mozilla_signed_extension': True,
'platform': amo.PLATFORM_ALL.id,
'size': 42,
'filename': 'notify-link-clicks-i18n.xpi',
'is_webextension': True
},
version_kw={
'license': license,
'min_app_version': '50.0',
'max_app_version': '*',
'release_notes': {
'en-US': u'Release notes in english',
'fr': u'Notes de version en français',
},
'reviewed': self.days_ago(0),
}
)
extract_version_to_git(self.addon.current_version.pk)
self.addon.current_version.reload()
assert self.addon.current_version.release_notes
self.version = self.addon.current_version
def extract_versions_to_git(addon_pk, version_pks):
log.info(
'Starting the git extraction of {} versions for add-on "{}".'.format(
len(version_pks), addon_pk))
for version_pk in version_pks:
extract_version_to_git(version_id=version_pk, force_extraction=True)
def migrate_webextensions_to_git_storage(ids, **kw):
# recursive imports...
from olympia.versions.tasks import (
extract_version_to_git, extract_version_source_to_git)
log.info(
'Migrating add-ons to git storage %d-%d [%d].',
ids[0], ids[-1], len(ids))
addons = Addon.unfiltered.filter(id__in=ids)
for addon in addons:
# Filter out versions that are already present in the git
# storage.
versions = addon.versions.filter(git_hash='').order_by('created')
for version in versions:
# Back in the days an add-on was able to have multiple files
# per version. That changed, we are very naive here and extracting
# simply the first file in the list. For WebExtensions there is
# only a very very small number that have different files for
# a single version.
unique_file_hashes = set([
x.original_hash for x in version.all_files
])
if len(unique_file_hashes) > 1:
# Log actually different hashes so that we can clean them
# up manually and work together with developers later.
log.info(
'Version {version} of {addon} has more than one uploaded '
'file'.format(version=repr(version), addon=repr(addon)))
if not unique_file_hashes:
log.info('No files found for {version} from {addon}'.format(
version=repr(version), addon=repr(addon)))
continue
# Don't call the task as a task but do the extraction in process
# this makes sure we don't overwhelm the storage and also makes
# sure we don't end up with tasks committing at random times but
# correctly in-order instead.
try:
file_id = version.all_files[0].pk
log.info('Extracting file {file_id} to git storage'.format(
file_id=file_id))
extract_version_to_git(version.pk)
if version.source:
extract_version_source_to_git(version.pk)
log.info(
'Extraction of file {file_id} into git storage succeeded'
.format(file_id=file_id))
except Exception:
log.exception(
'Extraction of file {file_id} from {version} '
'({addon}) failed'.format(
file_id=version.all_files[0],
version=repr(version),
addon=repr(addon)))
continue
