def migrate_webextensions_to_git_storage(ids, **kw): # recursive imports... from olympia.versions.tasks import extract_version_to_git log.info( 'Migrating add-ons to git storage %d-%d [%d].', ids[0], ids[-1], len(ids)) addons = Addon.unfiltered.filter(id__in=ids) for addon in addons: # Filter out versions that are already present in the git # storage. versions = addon.versions.filter(git_hash='').order_by('created') for version in versions: # Back in the days an add-on was able to have multiple files # per version. That changed, we are very naive here and extracting # simply the first file in the list. For WebExtensions there is # only a very very small number that have different files for # a single version. unique_file_hashes = set([ x.original_hash for x in version.all_files ]) if len(unique_file_hashes) > 1: # Log actually different hashes so that we can clean them # up manually and work together with developers later. log.info( 'Version {version} of {addon} has more than one uploaded ' 'file'.format(version=repr(version), addon=repr(addon))) if not unique_file_hashes: log.info('No files found for {version} from {addon}'.format( version=repr(version), addon=repr(addon))) continue # Don't call the task as a task but do the extraction in process # this makes sure we don't overwhelm the storage and also makes # sure we don't end up with tasks committing at random times but # correctly in-order instead. try: file_id = version.all_files[0].pk log.info('Extracting file {file_id} to git storage'.format( file_id=file_id)) extract_version_to_git(version.pk) log.info( 'Extraction of file {file_id} into git storage succeeded' .format(file_id=file_id)) except Exception: log.exception( 'Extraction of file {file_id} from {version} ' '({addon}) failed'.format( file_id=version.all_files[0], version=repr(version), addon=repr(addon))) continue
def test_extract_version_to_git_with_not_extension_type(): addon = addon_factory(type=amo.ADDON_STATICTHEME) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert not repo.is_extracted
def test_extract_version_to_git_with_non_webextension(): addon = addon_factory(type=amo.ADDON_EXTENSION, file_kw={'is_webextension': False}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert not repo.is_extracted
def setUp(self): super(TestFileEntriesSerializer, self).setUp() self.addon = addon_factory( file_kw={ 'filename': 'notify-link-clicks-i18n.xpi', 'is_webextension': True}) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db()
def setUp(self): super(TestAddonCompareVersionSerializerFileOnly, self).setUp() self.addon = addon_factory( file_kw={'filename': 'notify-link-clicks-i18n.xpi'}) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.reload() self.version = self.addon.current_version
def setUp(self): super(TestFileEntriesSerializer, self).setUp() self.addon = addon_factory( file_kw={ 'filename': 'notify-link-clicks-i18n.xpi', 'is_webextension': True}) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db()
def setUp(self): super(TestFileEntriesDiffSerializer, self).setUp() self.addon = addon_factory( name=u'My Addôn', slug='my-addon', file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db()
def test_extract_version_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_to_git_with_cron_enabled_and_force_extraction(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) repo = AddonGitRepository(addon.pk) create_switch('enable-git-extraction-cron') assert GitExtractionEntry.objects.count() == 0 extract_version_to_git(addon.current_version.pk, force_extraction=True) assert GitExtractionEntry.objects.count() == 0 assert repo.is_extracted
def test_extract_version_to_git(incr_mock, timer_mock): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) extract_version_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git'] timer_mock.assert_any_call('git.extraction.version') incr_mock.assert_called_with('git.extraction.version.success')
def setUp(self): super(TestAddonCompareVersionSerializer, self).setUp() self.addon = addon_factory(name=u'My Addôn', slug='my-addon', file_kw={ 'filename': 'webextension_no_id.xpi', 'is_webextension': True }) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db() self.version = self.addon.current_version
def setUp(self): super(TestFileInfoSerializer, self).setUp() self.addon = addon_factory(file_kw={ 'filename': 'notify-link-clicks-i18n.xpi', 'is_webextension': True }) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db() extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.reload() assert (self.addon.current_version.current_file.filename == 'notify-link-clicks-i18n.xpi') self.version = self.addon.current_version self.file = self.addon.current_version.current_file
def test_extract_version_to_git_deleted_version(): addon = addon_factory(file_kw={ 'filename': 'webextension_no_id.xpi', 'is_webextension': True }) version = addon.current_version version.delete() hide_disabled_files() extract_version_to_git(version.pk) repo = AddonGitRepository(addon.pk) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon') assert os.listdir(repo.git_repository_path) == ['.git']
def test_supports_search_plugin(self): self.addon = addon_factory(file_kw={'filename': 'search_20190331.xml'}) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.refresh_from_db() file = self.addon.current_version.current_file data = self.serialize(file) assert data['id'] == file.pk assert set(data['entries'].keys()) == {'search_20190331.xml'} assert data['selected_file'] == 'search_20190331.xml' assert data['content'].startswith( '<?xml version="1.0" encoding="utf-8"?>') assert data['download_url'] == absolutify( reverse('reviewers.download_git_file', kwargs={ 'version_id': self.addon.current_version.pk, 'filename': 'search_20190331.xml' }))
def create_new_version_for_addon(self, xpi_filename): addon = addon_factory(name=u'My Addôn', slug='my-addon', file_kw={'filename': xpi_filename}) extract_version_to_git(addon.current_version.pk) addon.current_version.refresh_from_db() parent_version = addon.current_version new_version = version_factory(addon=addon, file_kw={ 'filename': xpi_filename, 'is_webextension': True, }) repo = AddonGitRepository.extract_and_commit_from_version(new_version) return addon, repo, parent_version, new_version
def setUp(self): super(TestAddonBrowseVersionSerializer, self).setUp() license = License.objects.create( name={ 'en-US': u'My License', 'fr': u'Mä Licence', }, text={ 'en-US': u'Lorem ipsum dolor sit amet, has nemore patrioqué', }, url='http://license.example.com/' ) self.addon = addon_factory( file_kw={ 'hash': 'fakehash', 'is_mozilla_signed_extension': True, 'platform': amo.PLATFORM_ALL.id, 'size': 42, 'filename': 'notify-link-clicks-i18n.xpi', 'is_webextension': True }, version_kw={ 'license': license, 'min_app_version': '50.0', 'max_app_version': '*', 'release_notes': { 'en-US': u'Release notes in english', 'fr': u'Notes de version en français', }, 'reviewed': self.days_ago(0), } ) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.reload() assert self.addon.current_version.release_notes self.version = self.addon.current_version
def setUp(self): super(TestAddonBrowseVersionSerializer, self).setUp() license = License.objects.create( name={ 'en-US': u'My License', 'fr': u'Mä Licence', }, text={ 'en-US': u'Lorem ipsum dolor sit amet, has nemore patrioqué', }, url='http://license.example.com/' ) self.addon = addon_factory( file_kw={ 'hash': 'fakehash', 'is_mozilla_signed_extension': True, 'platform': amo.PLATFORM_ALL.id, 'size': 42, 'filename': 'notify-link-clicks-i18n.xpi', 'is_webextension': True }, version_kw={ 'license': license, 'min_app_version': '50.0', 'max_app_version': '*', 'release_notes': { 'en-US': u'Release notes in english', 'fr': u'Notes de version en français', }, 'reviewed': self.days_ago(0), } ) extract_version_to_git(self.addon.current_version.pk) self.addon.current_version.reload() assert self.addon.current_version.release_notes self.version = self.addon.current_version
def extract_versions_to_git(addon_pk, version_pks): log.info( 'Starting the git extraction of {} versions for add-on "{}".'.format( len(version_pks), addon_pk)) for version_pk in version_pks: extract_version_to_git(version_id=version_pk, force_extraction=True)
def migrate_webextensions_to_git_storage(ids, **kw): # recursive imports... from olympia.versions.tasks import ( extract_version_to_git, extract_version_source_to_git) log.info( 'Migrating add-ons to git storage %d-%d [%d].', ids[0], ids[-1], len(ids)) addons = Addon.unfiltered.filter(id__in=ids) for addon in addons: # Filter out versions that are already present in the git # storage. versions = addon.versions.filter(git_hash='').order_by('created') for version in versions: # Back in the days an add-on was able to have multiple files # per version. That changed, we are very naive here and extracting # simply the first file in the list. For WebExtensions there is # only a very very small number that have different files for # a single version. unique_file_hashes = set([ x.original_hash for x in version.all_files ]) if len(unique_file_hashes) > 1: # Log actually different hashes so that we can clean them # up manually and work together with developers later. log.info( 'Version {version} of {addon} has more than one uploaded ' 'file'.format(version=repr(version), addon=repr(addon))) if not unique_file_hashes: log.info('No files found for {version} from {addon}'.format( version=repr(version), addon=repr(addon))) continue # Don't call the task as a task but do the extraction in process # this makes sure we don't overwhelm the storage and also makes # sure we don't end up with tasks committing at random times but # correctly in-order instead. try: file_id = version.all_files[0].pk log.info('Extracting file {file_id} to git storage'.format( file_id=file_id)) extract_version_to_git(version.pk) if version.source: extract_version_source_to_git(version.pk) log.info( 'Extraction of file {file_id} into git storage succeeded' .format(file_id=file_id)) except Exception: log.exception( 'Extraction of file {file_id} from {version} ' '({addon}) failed'.format( file_id=version.all_files[0], version=repr(version), addon=repr(addon))) continue