def test_selected_file_unmodified(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
AddonGitRepository.extract_and_commit_from_version(new_version)
file = self.addon.current_version.current_file
data = self.serialize(file, parent_version=parent_version)
assert data['id'] == file.pk
# Unmodified file
manifest_data = data['entries']['manifest.json']
assert manifest_data['depth'] == 0
assert manifest_data['filename'] == u'manifest.json'
assert manifest_data['mime_category'] == 'text'
assert manifest_data['path'] == u'manifest.json'
assert manifest_data['status'] == ''
assert data['diff'] is not None
def setUp(self):
super().setUp()
self.addon = amo.tests.addon_factory(
file_kw={'filename': 'webextension.xpi'})
self.version = self.addon.current_version
responses.add_passthru(settings.AUTOGRAPH_CONFIG['server_url'])
# Make sure the initial version is already extracted, simulating
# a regular upload.
AddonGitRepository.extract_and_commit_from_version(self.version)
self.version.refresh_from_db()
def test_can_access_version_from_parent(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
AddonGitRepository.extract_and_commit_from_version(new_version)
serializer = AddonCompareVersionSerializer(
instance=new_version, context={'parent_version': parent_version})
file = serializer.data['file']
assert file['id'] == new_version.current_file.pk
def test_uses_unknown_minified_code(self):
parent_version = self.addon.current_version
new_version = version_factory(
addon=self.addon, file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
}
)
AddonGitRepository.extract_and_commit_from_version(new_version)
validation_data = {
'metadata': {
'unknownMinifiedFiles': ['README.md']
}
}
# Let's create a validation for the parent but not the current file
# which will result in us notifying the frontend of a minified file
# as well
current_validation = FileValidation.objects.create(
file=parent_version.current_file,
validation=json.dumps(validation_data))
self.version = new_version
self.file = new_version.current_file
data = self.serialize(parent_version=parent_version,
file='README.md')
assert data['uses_unknown_minified_code']
data = self.serialize(parent_version=parent_version,
file='manifest.json')
assert not data['uses_unknown_minified_code']
current_validation.delete()
# Creating a validation object for the current one works as well
FileValidation.objects.create(
file=self.version.current_file,
validation=json.dumps(validation_data))
data = self.serialize(
parent_version=parent_version, file='README.md')
assert data['uses_unknown_minified_code']
data = self.serialize(
parent_version=parent_version, file='manifest.json')
assert not data['uses_unknown_minified_code']
def test_extract_version_to_git_with_not_extension_type():
addon = addon_factory(type=amo.ADDON_STATICTHEME)
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert not repo.is_extracted
def test_serialize_deleted_file(self):
expected_filename = 'manifest.json'
parent_version = self.addon.current_version
new_version = version_factory(
addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
},
)
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, '', expected_filename, delete=True)
self.version = new_version
self.file = new_version.current_file
data = self.serialize(parent_version=parent_version)
assert data['download_url'] is None
# We deleted the selected file, so there should be a diff.
assert data['diff'] is not None
assert data['diff']['mode'] == 'D'
assert data['mimetype'] == 'application/json'
assert data['sha256'] is None
assert data['size'] is None
assert data['mime_category'] is None
assert data['filename'] == expected_filename
def test_extract_addon_with_more_versions_than_batch_size(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
version_1 = addon.current_version
version_2 = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository(addon)
entry = GitExtractionEntry.objects.create(addon=addon)
assert not version_1.git_hash
assert not version_2.git_hash
assert not repo.is_extracted
# First execution of the CRON task.
self.command.extract_addon(entry, batch_size=1)
version_1.refresh_from_db()
version_2.refresh_from_db()
entry.refresh_from_db()
assert repo.is_extracted
assert version_1.git_hash
# We only git-extracted the first version because of batch_size=1.
assert not version_2.git_hash
# We keep the entry and we set `in_progress` to `False` because we
# still need to extract the second version.
assert not entry.in_progress
assert GitExtractionEntry.objects.filter(pk=entry.pk).exists()
# Second execution of the CRON task.
self.command.extract_addon(entry, batch_size=1)
version_2.refresh_from_db()
assert repo.is_extracted
assert version_2.git_hash
assert not GitExtractionEntry.objects.filter(pk=entry.pk).exists()
def test_extract_addon_with_broken_ref_error_during_extraction(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
version = addon.current_version
repo = AddonGitRepository(addon)
# Force the creation of the git repository.
repo.git_repository
assert repo.is_extracted
# Set the "creation time" of the git repository to something older than
# 1 hour.
update_git_repo_creation_time(repo, time=datetime.datetime(2020, 1, 1))
# Create a broken ref, see:
# https://github.com/mozilla/addons-server/issues/13590
Path(f'{repo.git_repository_path}/.git/refs/heads/listed').touch()
entry = GitExtractionEntry.objects.create(addon=addon)
with pytest.raises(BrokenRefError):
self.command.extract_addon(entry)
version.refresh_from_db()
assert not repo.is_extracted
assert not GitExtractionEntry.objects.filter(pk=entry.pk).exists()
assert not version.git_hash
new_entry = GitExtractionEntry.objects.get(addon_id=addon.pk)
assert new_entry and new_entry.in_progress is None
def create_new_version_for_addon(self, xpi_filename):
addon = addon_factory(
name='My Addôn',
slug='my-addon',
file_kw={
'filename': xpi_filename,
'is_webextension': True
},
)
extract_version_to_git(addon.current_version.pk)
addon.current_version.refresh_from_db()
parent_version = addon.current_version
new_version = version_factory(
addon=addon,
file_kw={
'filename': xpi_filename,
'is_webextension': True,
},
)
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
return addon, repo, parent_version, new_version
def test_checks_creation_time_before_deleting_repo(self):
addon = addon_factory()
addon_repo = AddonGitRepository(addon)
# Create the git repo
addon_repo.git_repository
# We do not update the creation time of the git repository here so that
# it is less than 1 hour (because the git repository was created in
# this test case).
assert addon_repo.is_extracted
assert addon_repo.is_recent
# Simulate a git extraction in progress.
GitExtractionEntry.objects.create(addon_id=addon.pk, in_progress=True)
# This is the error raised by the task that extracts a version.
exc = MissingMasterBranchError('cannot find master branch')
on_extraction_error(request=None,
exc=exc,
traceback=None,
addon_pk=addon.pk)
# When the creation time of an add-on git repository is too recent (< 1
# hour ago), then we do not delete the repository because it might be
# an "extraction loop" problem.
assert addon_repo.is_extracted
# The task should remove the existing git extraction entry.
assert GitExtractionEntry.objects.filter(in_progress=None).count() == 0
# The task should NOT re-add the add-on to the git extraction queue.
assert GitExtractionEntry.objects.filter(in_progress=True).count() == 0
def test_handles_missing_master_branch(self):
addon = addon_factory()
addon_repo = AddonGitRepository(addon)
# Create the git repo
addon_repo.git_repository
update_git_repo_creation_time(addon_repo,
time=datetime.datetime(2019, 1, 1))
assert addon_repo.is_extracted
assert not addon_repo.is_recent
# Simulate a git extraction in progress.
GitExtractionEntry.objects.create(addon_id=addon.pk, in_progress=True)
# This is the error raised by the task that extracts a version.
exc = MissingMasterBranchError('cannot find master branch')
on_extraction_error(request=None,
exc=exc,
traceback=None,
addon_pk=addon.pk)
# The task should remove the git repository on
# MissingMasterBranchError.
assert not addon_repo.is_extracted
# The task should remove the existing git extraction entry.
assert GitExtractionEntry.objects.filter(in_progress=True).count() == 0
# The task should re-add the add-on to the git extraction queue.
assert GitExtractionEntry.objects.filter(in_progress=None).count() == 1
def test_selected_file_unmodified(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
AddonGitRepository.extract_and_commit_from_version(new_version)
self.version = new_version
self.file = new_version.current_file
data = self.serialize(parent_version=parent_version)
assert data['id'] == self.addon.current_version.current_file.pk
assert data['filename'] == u'manifest.json'
assert data['diff'] is not None
def test_extract_version_to_git_with_non_webextension():
addon = addon_factory(type=amo.ADDON_EXTENSION,
file_kw={'is_webextension': False})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert not repo.is_extracted
def test_basic(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
migrate_webextensions_to_git_storage([addon.pk])
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_basic(self):
expected_file_type = 'text'
expected_filename = 'manifest.json'
parent_version = self.addon.current_version
new_version = version_factory(
addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
},
)
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, 'Updated test file\n', 'test.txt')
apply_changes(repo, new_version, '', 'README.md', delete=True)
self.version = new_version
data = self.serialize(parent_version=parent_version)
assert set(data['file_entries'].keys()) == {
'manifest.json',
'README.md',
'test.txt',
}
# Unmodified file
manifest_data = data['file_entries']['manifest.json']
assert manifest_data['depth'] == 0
assert manifest_data['filename'] == expected_filename
assert manifest_data['mime_category'] == expected_file_type
assert manifest_data['path'] == 'manifest.json'
assert manifest_data['status'] == ''
# Added a new file
test_txt_data = data['file_entries']['test.txt']
assert test_txt_data['depth'] == 0
assert test_txt_data['filename'] == 'test.txt'
assert test_txt_data['mime_category'] == 'text'
assert test_txt_data['path'] == 'test.txt'
assert test_txt_data['status'] == 'A'
# Deleted file
readme_data = data['file_entries']['README.md']
assert readme_data['status'] == 'D'
assert readme_data['depth'] == 0
assert readme_data['filename'] == 'README.md'
# Not testing mimetype as text/markdown is missing in CI mimetypes
# database. But it doesn't matter much here since we're primarily
# after the git status.
assert readme_data['mime_category'] is None
assert readme_data['path'] == 'README.md'
def test_can_exclude_entries(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
AddonGitRepository.extract_and_commit_from_version(new_version)
file = self.addon.current_version.current_file
data = self.serialize(file,
exclude_entries=True,
parent_version=parent_version)
assert data['id'] == file.pk
assert data['selected_file'] == 'manifest.json'
assert data['mimetype'] == 'application/json'
assert data['entries'] is None
assert data['diff'] is not None
def test_extract_version_to_git_with_cron_enabled_and_force_extraction():
addon = addon_factory(file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
repo = AddonGitRepository(addon.pk)
create_switch('enable-git-extraction-cron')
assert GitExtractionEntry.objects.count() == 0
extract_version_to_git(addon.current_version.pk, force_extraction=True)
assert GitExtractionEntry.objects.count() == 0
assert repo.is_extracted
def test_can_exclude_entries_and_specify_a_file(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
AddonGitRepository.extract_and_commit_from_version(new_version)
file = self.addon.current_version.current_file
data = self.serialize(file,
file='README.md',
exclude_entries=True,
parent_version=parent_version)
assert data['id'] == file.pk
assert data['selected_file'] == 'README.md'
assert data['mimetype'] == 'text/markdown'
assert data['entries'] is None
assert data['diff'] is not None
def test_extract_addon(self):
version = self.addon.current_version
repo = AddonGitRepository(self.addon)
entry = GitExtractionEntry.objects.create(addon=self.addon)
assert not version.git_hash
assert not repo.is_extracted
self.command.extract_addon(entry)
version.refresh_from_db()
assert repo.is_extracted
assert not GitExtractionEntry.objects.filter(pk=entry.pk).exists()
assert version.git_hash
def test_basic(self):
expected_file_type = 'text'
expected_filename = 'manifest.json'
expected_mimetype = 'application/json'
expected_sha256 = (
'b634285d4b20bf6b198b2b2897c78b8e2c6eb39c92759025e338a14d18478dcb'
)
expected_size = 698
parent_version = self.addon.current_version
new_version = version_factory(
addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
},
)
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, 'Updated test file\n', 'test.txt')
apply_changes(repo, new_version, '', 'README.md', delete=True)
self.version = new_version
self.file = new_version.current_file
data = self.serialize(parent_version=parent_version)
assert data['id'] == new_version.current_file.pk
assert data['base_file'] == {'id': parent_version.current_file.pk}
assert data['selected_file'] == 'manifest.json'
assert data['download_url'] == absolutify(
reverse(
'reviewers.download_git_file',
kwargs={
'version_id': self.addon.current_version.pk,
'filename': 'manifest.json',
},
)
)
assert not data['uses_unknown_minified_code']
assert data['mimetype'] == expected_mimetype
assert data['sha256'] == expected_sha256
assert data['size'] == expected_size
assert data['mime_category'] == expected_file_type
assert data['filename'] == expected_filename
# The API always renders a diff, even for unmodified files.
assert data['diff'] is not None
def test_extract_version_to_git(incr_mock, timer_mock):
addon = addon_factory(file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
extract_version_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
timer_mock.assert_any_call('git.extraction.version')
incr_mock.assert_called_with('git.extraction.version.success')
def test_extract_addon(self):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
version = addon.current_version
repo = AddonGitRepository(addon)
entry = GitExtractionEntry.objects.create(addon=addon)
assert not version.git_hash
assert not repo.is_extracted
self.command.extract_addon(entry)
version.refresh_from_db()
assert repo.is_extracted
assert not GitExtractionEntry.objects.filter(pk=entry.pk).exists()
assert version.git_hash
def test_basic(self):
expected_file_type = 'text'
expected_filename = 'manifest.json'
expected_mimetype = 'application/json'
expected_sha256 = (
'bf9b0744c0011cad5caa55236951eda523f17676e91353a64a32353eac798631'
)
expected_size = 621
parent_version = self.addon.current_version
new_version = version_factory(
addon=self.addon, file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
}
)
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, 'Updated test file\n', 'test.txt')
apply_changes(repo, new_version, '', 'README.md', delete=True)
self.version = new_version
self.file = new_version.current_file
data = self.serialize(parent_version=parent_version)
assert data['id'] == new_version.current_file.pk
assert data['base_file'] == {
'id': parent_version.current_file.pk
}
assert data['selected_file'] == 'manifest.json'
assert data['download_url'] == absolutify(reverse(
'reviewers.download_git_file',
kwargs={
'version_id': self.addon.current_version.pk,
'filename': 'manifest.json'
}
))
assert not data['uses_unknown_minified_code']
assert data['mimetype'] == expected_mimetype
assert data['sha256'] == expected_sha256
assert data['size'] == expected_size
assert data['mime_category'] == expected_file_type
assert data['filename'] == expected_filename
# The API always renders a diff, even for unmodified files.
assert data['diff'] is not None
def test_with_generic_error(self):
addon = addon_factory()
addon_repo = AddonGitRepository(addon)
# Create the git repo
addon_repo.git_repository
assert addon_repo.is_extracted
# Simulate a git extraction in progress.
GitExtractionEntry.objects.create(addon_id=addon.pk, in_progress=True)
exc = Exception('some error')
on_extraction_error(
request=None, exc=exc, traceback=None, addon_pk=addon.pk
)
assert addon_repo.is_extracted
assert GitExtractionEntry.objects.count() == 0
def test_runs_git_extraction_after_signing(self):
old_git_hash = self.version.git_hash
with transaction.atomic():
signing.sign_file(self.version.current_file)
self.version.refresh_from_db()
assert self.version.git_hash != old_git_hash
repo = AddonGitRepository(self.addon)
output = _run_process('git log listed', repo)
assert output.count('Create new version') == 2
assert '(after successful signing)' in output
# 2 actual commits, including the repo initialization
assert output.count('Mozilla Add-ons Robot') == 3
def test_extract_version_to_git_deleted_version():
addon = addon_factory(file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True
})
version = addon.current_version
version.delete()
hide_disabled_files()
extract_version_to_git(version.pk)
repo = AddonGitRepository(addon.pk)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
def extract_version_to_git(version_id,
author_id=None,
note=None,
force_extraction=False):
"""Extract a `File` into our git storage backend."""
# We extract deleted or disabled versions as well so we need to make sure
# we can access them.
version = Version.unfiltered.get(pk=version_id)
if (version.addon.type != amo.ADDON_EXTENSION
or not version.all_files[0].is_webextension):
log.warning('Skipping git extraction of add-on "{}": not a '
'web-extension.'.format(version.addon.id))
return
if not force_extraction and waffle.switch_is_active(
'enable-git-extraction-cron'):
log.info('Adding add-on "{}" to the git extraction '
'queue.'.format(version.addon.id))
GitExtractionEntry.objects.create(addon=version.addon)
return
if author_id is not None:
author = UserProfile.objects.get(pk=author_id)
else:
author = None
log.info('Extracting {version_id} into git backend'.format(
version_id=version_id))
try:
with statsd.timer('git.extraction.version'):
repo = AddonGitRepository.extract_and_commit_from_version(
version=version, author=author, note=note)
statsd.incr('git.extraction.version.success')
except Exception as exc:
statsd.incr('git.extraction.version.failure')
raise exc
log.info('Extracted {version} into {git_path}'.format(
version=version_id, git_path=repo.git_repository_path))
def test_commits_to_git_async_signing_happened(self, extract_mock):
old_git_hash = self.version.git_hash
def call_sign_file():
signing.sign_file(self.version.current_file)
# raise ValueError after the sign_file call so that
# the extraction is queued via the on_commit hook
# but the atomic block won't complete.
raise ValueError()
with pytest.raises(ValueError):
with transaction.atomic():
call_sign_file()
extract_mock.assert_not_called()
self.version.refresh_from_db()
assert self.version.git_hash == old_git_hash
repo = AddonGitRepository(self.addon)
output = _run_process('git log listed', repo)
assert output.count('Create new version') == 1
def repo(self):
return AddonGitRepository(self.get_instance().version.addon)
def repo(self):
return AddonGitRepository(self._get_version().addon)