def test_extract_addon_continues_git_extraction(self, chain_mock): version1 = self.addon.current_version version2 = version_factory(addon=self.addon, ) version_factory(addon=self.addon, ) entry = GitExtractionEntry.objects.create(addon=self.addon) self.command.extract_addon(entry, batch_size=2) chain_mock.assert_called_with( extract_versions_to_git.si(addon_pk=self.addon.pk, version_pks=[version1.pk, version2.pk]), continue_git_extraction.si(self.addon.pk), )
def extract_addon(self, entry, batch_size=BATCH_SIZE): """ This method takes a GitExtractionEntry object and creates a chain of Celery tasks to extract each version in a git repository that haven't been extracted yet (including the deleted versions). It does not run if the add-on is locked for git extraction. """ addon = entry.addon log.info('Starting git extraction of add-on "%s".', addon.pk) # See: https://github.com/mozilla/addons-server/issues/14289 if addon.type != amo.ADDON_EXTENSION: log.info( 'Skipping git extraction of add-on "%s": not an extension.', addon.pk, ) entry.delete() return # We cannot use `entry.in_progress` because we have to be sure of the # add-on state and `entry` might not reflect the most up-to-date # database state here. if addon.git_extraction_is_in_progress: log.info( 'Aborting extraction of addon "%s" to git storage ' 'because it is already in progress.', addon.pk, ) return log.info('Locking add-on "%s" before extraction.', addon.pk) entry.update(in_progress=True) # Retrieve all the version pks to extract, sorted by creation date. versions_to_extract = (addon.versions( manager='unfiltered_for_relations').filter( file__is_webextension=True, git_hash='').order_by('created').values_list('pk', flat=True)) if len(versions_to_extract) == 0: log.info('No version to git-extract for add-on "%s", exiting.', addon.pk) # We can safely delete the entry because there is no version to # extract. entry.delete() return version_pks = versions_to_extract[0:batch_size] tasks = [ # Create a task to extract the BATCH_SIZE first versions. extract_versions_to_git.si(addon_pk=addon.pk, version_pks=version_pks) ] if len(version_pks) < len(versions_to_extract): # If there are more versions to git-extract, let's keep the entry # in the queue until we're done with this entry/add-on. The # `continue_git_extraction` task will set the `in_progress` flag to # `False` and this CRON task will pick the remaining versions to # git-extract the next time it runs. tasks.append(continue_git_extraction.si(addon.pk)) else: # If we do not have more versions to git-extract here, we can # remove the entry from the queue. tasks.append(remove_git_extraction_entry.si(addon.pk)) log.info( 'Submitted %s tasks to git-extract %s versions for add-on "%s".', len(tasks), len(versions_to_extract), addon.pk, ) # Attach an error handler on the chain and run it. The error # handler should remove the add-on lock (among other things). chain(*tasks).on_error(on_extraction_error.s(addon.pk)).delay()