def test_get_raw_diff_cache():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
# Let's a file
apply_changes(repo, version, '', 'manifest.json', delete=True)
with mock.patch('olympia.lib.git.pygit2.Repository.diff') as mocked_diff:
repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash)
repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash)
mocked_diff.assert_called_once()
assert (version.git_hash, original_version.git_hash) in repo._diff_cache
def test_get_diff_newline_both_no_newline():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
parent_version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(parent_version)
# Let's remove the newline
apply_changes(repo, parent_version, '{"id": "random"}', 'manifest.json')
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
# Now we're adding it again
apply_changes(repo, version, '{"id": "new random id"}', 'manifest.json')
changes = repo.get_diff(
commit=version.git_hash,
parent=parent_version.git_hash)
assert len(changes) == 1
assert changes[0]['new_ending_new_line'] is False
assert changes[0]['old_ending_new_line'] is False
def test_extract_and_commit_from_version(settings):
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
# Test that a new "unlisted" branch is created only if needed
addon.current_version.update(channel=amo.RELEASE_CHANNEL_UNLISTED)
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version)
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def test_get_diff_change_files_pathspec():
addon = addon_factory(file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
version = version_factory(
addon=addon, file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
apply_changes(repo, version, '{"id": "random"}\n', 'manifest.json')
apply_changes(repo, version, 'Updated readme\n', 'README.md')
changes = repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash,
pathspec=['README.md'])
assert len(changes) == 1
assert changes[0]
assert changes[0]['is_binary'] is False
assert changes[0]['lines_added'] == 1
assert changes[0]['lines_deleted'] == 25
assert changes[0]['mode'] == 'M'
assert changes[0]['old_path'] == 'README.md'
assert changes[0]['path'] == 'README.md'
assert changes[0]['size'] == 15
assert changes[0]['parent'] == original_version.git_hash
assert changes[0]['hash'] == version.git_hash
# There is actually just one big hunk in this diff since it's simply
# removing everything and adding new content
assert len(changes[0]['hunks']) == 1
assert changes[0]['hunks'][0]['header'] == '@@ -1,25 +1 @@'
assert changes[0]['hunks'][0]['old_start'] == 1
assert changes[0]['hunks'][0]['new_start'] == 1
assert changes[0]['hunks'][0]['old_lines'] == 25
assert changes[0]['hunks'][0]['new_lines'] == 1
hunk_changes = changes[0]['hunks'][0]['changes']
assert hunk_changes[0] == {
'content': '# notify-link-clicks-i18n',
'new_line_number': -1,
'old_line_number': 1,
'type': 'delete'
}
assert all(x['type'] == 'delete' for x in hunk_changes[:-1])
assert hunk_changes[-1] == {
'content': 'Updated readme',
'new_line_number': 1,
'old_line_number': -1,
'type': 'insert',
}
def test_extract_and_commit_from_version_set_git_hash():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
assert addon.current_version.git_hash == ''
AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version)
addon.current_version.refresh_from_db()
assert len(addon.current_version.git_hash) == 40
def test_extract_and_commit_source_from_version(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from source file'.format(
repr(addon.current_version), addon.current_version.id, repr(addon))
assert expected in output
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.git/config', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def test_get_diff_initial_commit_pathspec():
addon = addon_factory(file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
version = addon.current_version
repo = AddonGitRepository.extract_and_commit_from_version(version)
changes = repo.get_diff(
commit=version.git_hash,
parent=None,
pathspec=['_locales/de/messages.json'])
assert len(changes) == 1
# This makes sure that sub-directories are diffed properly too
assert changes[0]['is_binary'] is False
assert changes[0]['lines_added'] == 27
assert changes[0]['lines_deleted'] == 0
assert changes[0]['mode'] == 'A'
assert changes[0]['old_path'] == '_locales/de/messages.json'
assert changes[0]['parent'] == version.git_hash
assert changes[0]['hash'] == version.git_hash
assert changes[0]['path'] == '_locales/de/messages.json'
assert changes[0]['size'] == 658
# It's all an insert
assert all(
x['type'] == 'insert' for x in changes[0]['hunks'][0]['changes'])
def test_extract_and_commit_from_version_valid_extensions(settings, filename):
addon = addon_factory(file_kw={'filename': filename})
with mock.patch('olympia.files.utils.os.fsync') as fsync_mock:
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
# Make sure we are always calling fsync after extraction
assert fsync_mock.called
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
assert 'unlisted' not in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def test_extract_and_commit_from_version_multiple_versions(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'addon')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
# Create two more versions, check that they appear in the comitlog
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'},
version='0.2')
AddonGitRepository.extract_and_commit_from_version(version=version)
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'},
version='0.3')
repo = AddonGitRepository.extract_and_commit_from_version(version=version)
output = _run_process('git log listed', repo)
assert output.count('Create new version') == 3
assert '0.1' in output
assert '0.2' in output
assert '0.3' in output
# 4 actual commits, including the repo initialization
assert output.count('Mozilla Add-ons Robot') == 4
# Make sure the commits didn't spill over into the master branch
output = _run_process('git log', repo)
assert output.count('Mozilla Add-ons Robot') == 1
assert '0.1' not in output
def test_get_diff_add_new_file():
addon = addon_factory(file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
version = version_factory(
addon=addon, file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
apply_changes(repo, version, '{"id": "random"}\n', 'new_file.json')
changes = repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash)
assert changes[0]['hunks'] == [{
'changes': [{
'content': '{"id": "random"}',
'new_line_number': 1,
'old_line_number': -1,
'type': 'insert'
}],
'new_lines': 1,
'new_start': 1,
'old_lines': 0,
'old_start': 0,
'header': '@@ -0,0 +1 @@',
}]
assert changes[0]['is_binary'] is False
assert changes[0]['lines_added'] == 1
assert changes[0]['lines_deleted'] == 0
assert changes[0]['mode'] == 'A'
assert changes[0]['old_path'] == 'new_file.json'
assert changes[0]['path'] == 'new_file.json'
assert changes[0]['size'] == 17
assert changes[0]['parent'] == original_version.git_hash
assert changes[0]['hash'] == version.git_hash
def test_get_diff_delete_file():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
# Let's a file
apply_changes(repo, version, '', 'manifest.json', delete=True)
changes = repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash)
assert changes[0]['mode'] == 'D'
assert all(
x['type'] == 'delete' for x in changes[0]['hunks'][0]['changes'])
def test_extract_and_commit_from_version_commits_files(
settings, filename, expected):
addon = addon_factory(file_kw={'filename': filename})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process(
'git ls-tree -r --name-only listed:extracted', repo)
assert set(output.split()) == expected
def test_extract_and_commit_from_version_use_addons_robot_default():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version)
output = _run_process('git log --format=full listed', repo)
assert (
'Author: Mozilla Add-ons Robot '
'<*****@*****.**>'
in output)
assert (
'Commit: Mozilla Add-ons Robot '
'<*****@*****.**>'
in output)
def test_extract_and_commit_from_version_use_applied_author():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
user = user_factory(
email='*****@*****.**', display_name='Fancy Test User')
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version,
author=user)
output = _run_process('git log --format=full listed', repo)
assert 'Author: Fancy Test User <*****@*****.**>' in output
assert (
'Commit: Mozilla Add-ons Robot '
'<*****@*****.**>'
in output)
def test_extract_and_commit_from_version_support_custom_note():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
version=addon.current_version,
note='via signing')
output = _run_process('git log --format=full listed', repo)
print(output)
expected = (
'Create new version {} ({}) for {} from {} (via signing)'
.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0])))
assert expected in output
def test_extract_and_commit_from_version_reverts_active_locale():
from django.utils.translation import get_language
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
with activate_locale('fr'):
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
assert get_language() == 'fr'
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from {}'.format(
repr(addon.current_version), addon.current_version.id, repr(addon),
repr(addon.current_version.all_files[0]))
assert expected in output
def extract_version_to_git(version_id, author_id=None):
"""Extract a `File` into our git storage backend."""
# We extract deleted or disabled versions as well so we need to make sure
# we can access them.
version = Version.unfiltered.get(pk=version_id)
if author_id is not None:
author = UserProfile.objects.get(pk=author_id)
else:
author = None
log.info('Extracting {version_id} into git backend'.format(
version_id=version_id))
repo = AddonGitRepository.extract_and_commit_from_version(
version=version, author=author)
log.info('Extracted {version} into {git_path}'.format(
version=version_id, git_path=repo.git_repository_path))
def test_iter_tree():
addon = addon_factory(file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(
addon.current_version)
commit = repo.git_repository.revparse_single('listed')
tree = list(repo.iter_tree(repo.get_root_tree(commit)))
# path, filename mapping
expected_files = [
('README.md', 'README.md', 'blob'),
('_locales', '_locales', 'tree'),
('_locales/de', 'de', 'tree'),
('_locales/de/messages.json', 'messages.json', 'blob'),
('_locales/en', 'en', 'tree'),
('_locales/en/messages.json', 'messages.json', 'blob'),
('_locales/ja', 'ja', 'tree'),
('_locales/ja/messages.json', 'messages.json', 'blob'),
('_locales/nb_NO', 'nb_NO', 'tree'),
('_locales/nb_NO/messages.json', 'messages.json', 'blob'),
('_locales/nl', 'nl', 'tree'),
('_locales/nl/messages.json', 'messages.json', 'blob'),
('_locales/ru', 'ru', 'tree'),
('_locales/ru/messages.json', 'messages.json', 'blob'),
('_locales/sv', 'sv', 'tree'),
('_locales/sv/messages.json', 'messages.json', 'blob'),
('background-script.js', 'background-script.js', 'blob'),
('content-script.js', 'content-script.js', 'blob'),
('icons', 'icons', 'tree'),
('icons/LICENSE', 'LICENSE', 'blob'),
('icons/link-48.png', 'link-48.png', 'blob'),
('manifest.json', 'manifest.json', 'blob'),
]
for idx, entry in enumerate(tree):
expected_path, expected_name, expected_type = expected_files[idx]
assert entry.path == expected_path
assert entry.tree_entry.name == expected_name
assert entry.tree_entry.type == expected_type
def extract_version_source_to_git(version_id, author_id=None):
# We extract deleted or disabled versions as well so we need to make sure
# we can access them.
version = Version.unfiltered.get(pk=version_id)
if not version.source:
log.info('Tried to extract sources of {version_id} but there none.')
return
if author_id is not None:
author = UserProfile.objects.get(pk=author_id)
else:
author = None
log.info('Extracting {version_id} source into git backend'.format(
version_id=version_id))
repo = AddonGitRepository.extract_and_commit_source_from_version(
version=version, author=author)
log.info(
'Extracted source files from {version} into {git_path}'.format(
version=version_id, git_path=repo.git_repository_path))
def test_extract_and_commit_source_from_version_rename_dotgit_files(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.gitattributes', '')
zip_file.writestr('.gitignore', '')
zip_file.writestr('.gitmodules', '')
zip_file.writestr('some/directory/.gitattributes', '')
zip_file.writestr('some/directory/.gitignore', '')
zip_file.writestr('some/directory/.gitmodules', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json',
'extracted/.gitattributes.b236f599',
'extracted/.gitignore.b236f599',
'extracted/.gitmodules.b236f599',
'extracted/some/directory/.gitattributes.b236f599',
'extracted/some/directory/.gitignore.b236f599',
'extracted/some/directory/.gitmodules.b236f599',
}
def from_upload(cls, upload, addon, selected_apps, channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
assert parsed_data is not None
from olympia.addons.models import AddonFeatureCompatibility
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approvalnotes = None
if parsed_data.get('is_mozilla_signed_extension'):
approvalnotes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approvalnotes=approvalnotes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = parsed_data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0]
)
feature_compatibility.update(e10s=e10s_compatibility)
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [File.from_upload(
upload=upload, version=version, platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data
)]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
version_uploaded.send(sender=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Extract into git repository
AddonGitRepository.extract_and_commit_from_file_obj(
file_obj=version.all_files[0],
channel=channel,
author=upload.user)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def test_get_diff_newline_both_no_newline():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
parent_version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(parent_version)
# Let's remove the newline
apply_changes(repo, parent_version, '{"id": "random"}', 'manifest.json')
version = version_factory(
addon=addon, file_kw={'filename': 'webextension_no_id.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
# And create another change that doesn't add a newline
apply_changes(
repo, version,
'{"id": "new random id",\n"something": "foo"}',
'manifest.json')
changes = repo.get_diff(
commit=version.git_hash,
parent=parent_version.git_hash)
assert len(changes) == 1
assert changes[0]['new_ending_new_line'] is False
assert changes[0]['old_ending_new_line'] is False
hunk_changes = changes[0]['hunks'][0]['changes']
# The following structure represents a diff similar to this one
#
# diff --git a/manifest.json b/manifest.json
# index 72bd4f0..1f666c8 100644
# --- a/manifest.json
# +++ b/manifest.json
# @@ -1 +1,2 @@
# -{"id": "random"}
# \ No newline at end of file
# +{"id": "new random id",
# +"something": "foo"}
# \ No newline at end of file
assert hunk_changes == [
{
'content': '{"id": "random"}',
'new_line_number': -1,
'old_line_number': 1,
'type': 'delete'
},
{
'content': '\n\\ No newline at end of file',
'new_line_number': -1,
'old_line_number': 1,
'type': 'insert-eofnl'},
{
'content': '{"id": "new random id",',
'new_line_number': 1,
'old_line_number': -1,
'type': 'insert'
},
{
'content': '"something": "foo"}',
'new_line_number': 2,
'old_line_number': -1,
'type': 'insert'
},
{
'content': '\n\\ No newline at end of file',
'new_line_number': 2,
'old_line_number': -1,
'type': 'delete-eofnl'
}
]
def test_get_diff_change_files():
addon = addon_factory(file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
original_version = addon.current_version
AddonGitRepository.extract_and_commit_from_version(original_version)
version = version_factory(
addon=addon, file_kw={'filename': 'notify-link-clicks-i18n.xpi'})
repo = AddonGitRepository.extract_and_commit_from_version(version)
apply_changes(repo, version, '{"id": "random"}\n', 'manifest.json')
apply_changes(repo, version, 'Updated readme\n', 'README.md')
changes = repo.get_diff(
commit=version.git_hash,
parent=original_version.git_hash)
assert len(changes) == 2
assert changes[0]
assert changes[0]['is_binary'] is False
assert changes[0]['lines_added'] == 1
assert changes[0]['lines_deleted'] == 25
assert changes[0]['mode'] == 'M'
assert changes[0]['old_path'] == 'README.md'
assert changes[0]['path'] == 'README.md'
assert changes[0]['size'] == 15
assert changes[0]['parent'] == original_version.git_hash
assert changes[0]['hash'] == version.git_hash
# There is actually just one big hunk in this diff since it's simply
# removing everything and adding new content
assert len(changes[0]['hunks']) == 1
assert changes[0]['hunks'][0]['header'] == '@@ -1,25 +1 @@'
assert changes[0]['hunks'][0]['old_start'] == 1
assert changes[0]['hunks'][0]['new_start'] == 1
assert changes[0]['hunks'][0]['old_lines'] == 25
assert changes[0]['hunks'][0]['new_lines'] == 1
hunk_changes = changes[0]['hunks'][0]['changes']
assert hunk_changes[0] == {
'content': '# notify-link-clicks-i18n',
'new_line_number': -1,
'old_line_number': 1,
'type': 'delete'
}
assert all(x['type'] == 'delete' for x in hunk_changes[:-1])
assert hunk_changes[-1] == {
'content': 'Updated readme',
'new_line_number': 1,
'old_line_number': -1,
'type': 'insert',
}
assert changes[1]
assert changes[1]['is_binary'] is False
assert changes[1]['lines_added'] == 1
assert changes[1]['lines_deleted'] == 32
assert changes[1]['mode'] == 'M'
assert changes[1]['old_path'] == 'manifest.json'
assert changes[1]['path'] == 'manifest.json'
assert changes[1]['size'] == 17
assert changes[1]['parent'] == original_version.git_hash
assert changes[1]['hash'] == version.git_hash
# There is actually just one big hunk in this diff since it's simply
# removing everything and adding new content
assert len(changes[1]['hunks']) == 1
assert changes[1]['hunks'][0]['header'] == '@@ -1,32 +1 @@'
assert changes[1]['hunks'][0]['old_start'] == 1
assert changes[1]['hunks'][0]['new_start'] == 1
assert changes[1]['hunks'][0]['old_lines'] == 32
assert changes[1]['hunks'][0]['new_lines'] == 1
hunk_changes = changes[1]['hunks'][0]['changes']
assert all(x['type'] == 'delete' for x in hunk_changes[:-1])
assert hunk_changes[-1] == {
'content': '{"id": "random"}',
'new_line_number': 1,
'old_line_number': -1,
'type': 'insert'
}
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
log.info('New version: %r (%s) from %r' %
(version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(upload=upload,
version=version,
platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
version_uploaded.send(sender=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Extract into git repository
AddonGitRepository.extract_and_commit_from_version(
version=version, author=upload.user)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def repo(self):
return AddonGitRepository(self.get_instance().version.addon)
def test_basic(self):
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, 'Updated test file\n', 'test.txt')
apply_changes(repo, new_version, '', 'README.md', delete=True)
file = self.addon.current_version.current_file
data = self.serialize(file, parent_version=parent_version)
assert data['id'] == file.pk
assert data['status'] == 'public'
assert data['hash'] == ''
assert data['is_webextension'] is True
assert data['created'] == (
file.created.replace(microsecond=0).isoformat() + 'Z')
assert data['url'] == ('http://testserver/firefox/downloads/file/{}'
'/webextension_no_id.xpi?src=').format(file.pk)
assert data['selected_file'] == 'manifest.json'
assert data['download_url'] == absolutify(
reverse('reviewers.download_git_file',
kwargs={
'version_id': self.addon.current_version.pk,
'filename': 'manifest.json'
}))
assert set(data['entries'].keys()) == {
'manifest.json', 'README.md', 'test.txt'
}
# Unmodified file
manifest_data = data['entries']['manifest.json']
assert manifest_data['depth'] == 0
assert manifest_data['filename'] == u'manifest.json'
assert manifest_data['sha256'] == (
'bf9b0744c0011cad5caa55236951eda523f17676e91353a64a32353eac798631')
assert manifest_data['mimetype'] == 'application/json'
assert manifest_data['mime_category'] == 'text'
assert manifest_data['path'] == u'manifest.json'
assert manifest_data['size'] == 621
assert manifest_data['status'] == ''
assert isinstance(manifest_data['modified'], datetime)
# Added a new file
test_txt_data = data['entries']['test.txt']
assert test_txt_data['depth'] == 0
assert test_txt_data['filename'] == u'test.txt'
assert test_txt_data['sha256'] == (
'f8b40fc302692ea4f552cb3d60bc89dd8b4616e398de5585e471cee73e2c0618')
assert test_txt_data['mimetype'] == 'text/plain'
assert test_txt_data['mime_category'] == 'text'
assert test_txt_data['path'] == u'test.txt'
assert test_txt_data['size'] == 18
assert test_txt_data['status'] == 'A'
# Deleted file
readme_data = data['entries']['README.md']
assert readme_data['status'] == 'D'
assert readme_data['depth'] == 0
assert readme_data['filename'] == 'README.md'
assert readme_data['sha256'] is None
# Not testing mimetype as text/markdown is missing in travis mimetypes
# database. But it doesn't matter much here since we're primarily
# after the git status.
assert readme_data['mime_category'] is None
assert readme_data['path'] == u'README.md'
assert readme_data['size'] is None
assert readme_data['modified'] is None
def test_basic(self):
expected_file_type = 'text'
expected_filename = 'manifest.json'
expected_mimetype = 'application/json'
expected_sha256 = (
'bf9b0744c0011cad5caa55236951eda523f17676e91353a64a32353eac798631')
expected_size = 621
parent_version = self.addon.current_version
new_version = version_factory(addon=self.addon,
file_kw={
'filename': 'webextension_no_id.xpi',
'is_webextension': True,
})
repo = AddonGitRepository.extract_and_commit_from_version(new_version)
apply_changes(repo, new_version, 'Updated test file\n', 'test.txt')
apply_changes(repo, new_version, '', 'README.md', delete=True)
file = self.addon.current_version.current_file
data = self.serialize(file, parent_version=parent_version)
assert data['id'] == file.pk
assert data['base_file'] == {'id': parent_version.current_file.pk}
assert data['status'] == 'public'
assert data['hash'] == ''
assert data['is_webextension'] is True
assert data['created'] == (
file.created.replace(microsecond=0).isoformat() + 'Z')
assert data['url'] == ('http://testserver/firefox/downloads/file/{}'
'/webextension_no_id.xpi?src=').format(file.pk)
assert data['selected_file'] == 'manifest.json'
assert data['download_url'] == absolutify(
reverse('reviewers.download_git_file',
kwargs={
'version_id': self.addon.current_version.pk,
'filename': 'manifest.json'
}))
assert not data['uses_unknown_minified_code']
assert data['mimetype'] == expected_mimetype
assert data['sha256'] == expected_sha256
assert data['size'] == expected_size
assert data['mime_category'] == expected_file_type
assert data['filename'] == expected_filename
assert set(data['entries'].keys()) == {
'manifest.json', 'README.md', 'test.txt'
}
# The API always renders a diff, even for unmodified files.
assert data['diff'] is not None
assert not data['uses_unknown_minified_code']
# Unmodified file
manifest_data = data['entries']['manifest.json']
assert manifest_data['depth'] == 0
assert manifest_data['filename'] == expected_filename
assert manifest_data['mime_category'] == expected_file_type
assert manifest_data['path'] == u'manifest.json'
assert manifest_data['status'] == ''
# Added a new file
test_txt_data = data['entries']['test.txt']
assert test_txt_data['depth'] == 0
assert test_txt_data['filename'] == u'test.txt'
assert test_txt_data['mime_category'] == 'text'
assert test_txt_data['path'] == u'test.txt'
assert test_txt_data['status'] == 'A'
# Deleted file
readme_data = data['entries']['README.md']
assert readme_data['status'] == 'D'
assert readme_data['depth'] == 0
assert readme_data['filename'] == 'README.md'
# Not testing mimetype as text/markdown is missing in travis mimetypes
# database. But it doesn't matter much here since we're primarily
# after the git status.
assert readme_data['mime_category'] is None
assert readme_data['path'] == u'README.md'
