def test_extract_addon_with_mock(self, chain_mock):
addon = addon_factory()
entry = GitExtractionEntry.objects.create(addon=addon)
self.command.extract_addon(entry)
chain_mock.assert_called_with(
extract_versions_to_git.si(addon_pk=addon.pk,
version_pks=[addon.current_version.pk]),
remove_git_extraction_entry.si(addon.pk),
)
def test_extract_addon_continues_git_extraction(self, chain_mock):
version1 = self.addon.current_version
version2 = version_factory(addon=self.addon, )
version_factory(addon=self.addon, )
entry = GitExtractionEntry.objects.create(addon=self.addon)
self.command.extract_addon(entry, batch_size=2)
chain_mock.assert_called_with(
extract_versions_to_git.si(addon_pk=self.addon.pk,
version_pks=[version1.pk, version2.pk]),
continue_git_extraction.si(self.addon.pk),
)
def test_extract_addon_called_more_than_once(self, chain_mock):
addon = addon_factory()
entry1 = GitExtractionEntry.objects.create(addon=addon)
entry2 = GitExtractionEntry.objects.create(addon=addon)
self.command.extract_addon(entry1)
self.command.extract_addon(entry2)
chain_mock.assert_called_with(
extract_versions_to_git.si(addon_pk=addon.pk,
version_pks=[addon.current_version.pk]),
remove_git_extraction_entry.si(addon.pk),
)
chain_mock.call_count == 1
def extract_addon(self, entry):
"""
This method takes a GitExtractionEntry object and creates a chain of
Celery tasks to extract each version in a git repository that haven't
been extracted yet (including the deleted versions).
It does not run if the add-on is locked for git extraction.
"""
addon = entry.addon
log.info('Starting git extraction of add-on "{}".'.format(addon.pk))
# We cannot use `entry.in_progress` because we have to be sure of the
# add-on state and `entry` might not reflect the most up-to-date
# database state here.
if addon.git_extraction_is_in_progress:
log.info('Aborting extraction of addon "{}" to git storage '
'because it is already in progress.'.format(addon.pk))
return
log.info('Locking add-on "{}" before extraction.'.format(addon.pk))
entry.update(in_progress=True)
# Retrieve all the version pks to extract, sorted by creation date.
versions_to_extract = (addon.versions(
manager='unfiltered_for_relations').filter(
git_hash='').order_by('created').values_list('pk', flat=True))
if len(versions_to_extract) == 0:
log.info('No version to git-extract for add-on "{}", '
'exiting.'.format(addon.pk))
# We can safely delete the entry because there is no version to
# extract.
entry.delete()
return
tasks = []
for version_pks in chunked(versions_to_extract, BATCH_SIZE):
# Create a task to extract a subset of versions.
tasks.append(
extract_versions_to_git.si(addon_pk=addon.pk,
version_pks=version_pks))
tasks.append(remove_git_extraction_entry.si(addon.pk))
log.info('Submitted {} tasks to git-extract {} versions for add-on '
'"{}".'.format(len(tasks), len(versions_to_extract),
addon.pk))
# Attach an error handler on the chain and run it. The error
# handler should remove the add-on lock (among other things).
chain(*tasks).on_error(on_extraction_error.s(addon.pk)).delay()
def test_extract_addon_with_multiple_versions(self, chain_mock):
addon = addon_factory()
version1 = addon.current_version
version2 = version_factory(addon=addon)
version_deleted = version_factory(addon=addon, deleted=True)
entry = GitExtractionEntry.objects.create(addon=addon)
self.command.extract_addon(entry)
chain_mock.assert_called_with(
extract_versions_to_git.si(
addon_pk=addon.pk,
version_pks=[version1.pk, version2.pk, version_deleted.pk],
),
remove_git_extraction_entry.si(addon.pk),
)
def test_extract_addon_with_multiple_versions(self, chain_mock):
version1 = self.addon.current_version
version2 = version_factory(addon=self.addon,
file_kw={'is_webextension': True})
version_deleted = version_factory(addon=self.addon,
deleted=True,
file_kw={'is_webextension': True})
# This version should be ignored because it is not a web-extension.
version_factory(addon=self.addon, file_kw={'is_webextension': False})
entry = GitExtractionEntry.objects.create(addon=self.addon)
self.command.extract_addon(entry)
chain_mock.assert_called_with(
extract_versions_to_git.si(
addon_pk=self.addon.pk,
version_pks=[version1.pk, version2.pk, version_deleted.pk],
),
remove_git_extraction_entry.si(self.addon.pk),
)
def extract_addon(self, entry, batch_size=BATCH_SIZE):
"""
This method takes a GitExtractionEntry object and creates a chain of
Celery tasks to extract each version in a git repository that haven't
been extracted yet (including the deleted versions).
It does not run if the add-on is locked for git extraction.
"""
addon = entry.addon
log.info('Starting git extraction of add-on "%s".', addon.pk)
# See: https://github.com/mozilla/addons-server/issues/14289
if addon.type != amo.ADDON_EXTENSION:
log.info(
'Skipping git extraction of add-on "%s": not an extension.',
addon.pk,
)
entry.delete()
return
# We cannot use `entry.in_progress` because we have to be sure of the
# add-on state and `entry` might not reflect the most up-to-date
# database state here.
if addon.git_extraction_is_in_progress:
log.info(
'Aborting extraction of addon "%s" to git storage '
'because it is already in progress.',
addon.pk,
)
return
log.info('Locking add-on "%s" before extraction.', addon.pk)
entry.update(in_progress=True)
# Retrieve all the version pks to extract, sorted by creation date.
versions_to_extract = (addon.versions(
manager='unfiltered_for_relations').filter(
file__is_webextension=True,
git_hash='').order_by('created').values_list('pk', flat=True))
if len(versions_to_extract) == 0:
log.info('No version to git-extract for add-on "%s", exiting.',
addon.pk)
# We can safely delete the entry because there is no version to
# extract.
entry.delete()
return
version_pks = versions_to_extract[0:batch_size]
tasks = [
# Create a task to extract the BATCH_SIZE first versions.
extract_versions_to_git.si(addon_pk=addon.pk,
version_pks=version_pks)
]
if len(version_pks) < len(versions_to_extract):
# If there are more versions to git-extract, let's keep the entry
# in the queue until we're done with this entry/add-on. The
# `continue_git_extraction` task will set the `in_progress` flag to
# `False` and this CRON task will pick the remaining versions to
# git-extract the next time it runs.
tasks.append(continue_git_extraction.si(addon.pk))
else:
# If we do not have more versions to git-extract here, we can
# remove the entry from the queue.
tasks.append(remove_git_extraction_entry.si(addon.pk))
log.info(
'Submitted %s tasks to git-extract %s versions for add-on "%s".',
len(tasks),
len(versions_to_extract),
addon.pk,
)
# Attach an error handler on the chain and run it. The error
# handler should remove the add-on lock (among other things).
chain(*tasks).on_error(on_extraction_error.s(addon.pk)).delay()
